From 6be4bb366913db6e9f9b8cc49ff07dcb96664661 Mon Sep 17 00:00:00 2001 From: "Simon A. Eugster" Date: Sun, 12 Feb 2012 16:57:31 +0100 Subject: [PATCH] Volume envelope calculation The AudioInfo class reads audio information from a MLT producer for easy access to sampling rate etc. AudioOffset now generates the volume envelope of an audio file and saves it as image. --- src/clipitem.h | 2 +- testingArea/CMakeLists.txt | 2 +- testingArea/audioInfo.cpp | 59 ++++++++++ testingArea/audioInfo.h | 35 ++++++ testingArea/audioOffset.cpp | 186 ++++++++++++++++---------------- testingArea/audioStreamInfo.cpp | 66 ++++++++++++ testingArea/audioStreamInfo.h | 44 ++++++++ 7 files changed, 296 insertions(+), 98 deletions(-) create mode 100644 testingArea/audioInfo.cpp create mode 100644 testingArea/audioInfo.h create mode 100644 testingArea/audioStreamInfo.cpp create mode 100644 testingArea/audioStreamInfo.h diff --git a/src/clipitem.h b/src/clipitem.h index 5629743f..c2870993 100644 --- a/src/clipitem.h +++ b/src/clipitem.h @@ -39,7 +39,7 @@ class Transition; namespace Mlt { class Producer; -}; +} class ClipItem : public AbstractClipItem { diff --git a/testingArea/CMakeLists.txt b/testingArea/CMakeLists.txt index 1b863b16..d08ae2b1 100644 --- a/testingArea/CMakeLists.txt +++ b/testingArea/CMakeLists.txt @@ -7,7 +7,7 @@ include_directories( ) include(${QT_USE_FILE}) -add_executable(audioOffset audioOffset.cpp) +add_executable(audioOffset audioOffset.cpp audioInfo.cpp audioStreamInfo.cpp) target_link_libraries(audioOffset ${QT_LIBRARIES} ${LIBMLT_LIBRARY} diff --git a/testingArea/audioInfo.cpp b/testingArea/audioInfo.cpp new file mode 100644 index 00000000..f35a8a9f --- /dev/null +++ b/testingArea/audioInfo.cpp @@ -0,0 +1,59 @@ +/*************************************************************************** + * Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com) * + * This file is part of kdenlive. See www.kdenlive.org. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + ***************************************************************************/ + +#include "audioInfo.h" + +#include "audioStreamInfo.h" +#include +#include + +AudioInfo::AudioInfo(Mlt::Producer *producer) +{ + // Since we already receive an MLT producer, we do not need to initialize MLT: + // Mlt::Factory::init(NULL); + + // Get the number of streams and add the information of each of them if it is an audio stream. + int streams = atoi(producer->get("meta.media.nb_streams")); + for (int i = 0; i < streams; i++) { + std::string propertyName = QString("meta.media.%1.stream.type").arg(i).toStdString(); + + if (strcmp("audio", producer->get(propertyName.c_str())) == 0) { + m_list << new AudioStreamInfo(producer, i); + } + + } +} + +AudioInfo::~AudioInfo() +{ + foreach (AudioStreamInfo *info, m_list) { + delete info; + } +} + +int AudioInfo::size() const +{ + return m_list.size(); +} + +AudioStreamInfo const* AudioInfo::info(int pos) const +{ + Q_ASSERT(pos >= 0); + Q_ASSERT(pos <= m_list.size()); + + return m_list.at(pos); +} + +void AudioInfo::dumpInfo() const +{ + foreach (AudioStreamInfo *info, m_list) { + info->dumpInfo(); + } +} diff --git a/testingArea/audioInfo.h b/testingArea/audioInfo.h new file mode 100644 index 00000000..5a30fcdd --- /dev/null +++ b/testingArea/audioInfo.h @@ -0,0 +1,35 @@ +/*************************************************************************** + * Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com) * + * This file is part of kdenlive. See www.kdenlive.org. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + ***************************************************************************/ + +#ifndef AUDIOINFO_H +#define AUDIOINFO_H + +#include +#include + +class AudioStreamInfo; +class AudioInfo +{ +public: + AudioInfo(Mlt::Producer *producer); + ~AudioInfo(); + + int size() const; + AudioStreamInfo const* info(int pos) const; + + void dumpInfo() const; + +private: + Mlt::Producer *m_producer; + QList m_list; + +}; + +#endif // AUDIOINFO_H diff --git a/testingArea/audioOffset.cpp b/testingArea/audioOffset.cpp index 2f8351a8..d8e8eede 100644 --- a/testingArea/audioOffset.cpp +++ b/testingArea/audioOffset.cpp @@ -1,63 +1,27 @@ +/*************************************************************************** + * Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com) * + * This file is part of kdenlive. See www.kdenlive.org. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + ***************************************************************************/ + #include #include +#include +#include +#include +#include #include #include #include +#include -int info(char *filename) -{ - - // Initialize MLT - Mlt::Factory::init(NULL); - - // Load an arbitrary profile - Mlt::Profile prof("hdv_1080_25p"); - - - std::cout << "MLT initialized, profile loaded. Loading clips ..." << std::endl; - - - Mlt::Producer producer(prof, filename); - if (!producer.is_valid()) { - std::cout << filename << " is invalid." << std::endl; - return 2; - } - std::cout << "Successfully loaded producer " << filename << std::endl; - - int streams = atoi(producer.get("meta.media.nb_streams")); - std::cout << "Number of streams: " << streams << std::endl; - - int audioStream = -1; - for (int i = 0; i < streams; i++) { - std::string propertyName = QString("meta.media.%1.stream.type").arg(i).toStdString(); - std::cout << "Producer " << i << ": " << producer.get(propertyName.c_str()); - if (strcmp("audio", producer.get(propertyName.c_str())) == 0) { - std::cout << " (Using this stream)"; - audioStream = i; - } - std::cout << std::endl; - } - - if (audioStream >= 0) { - QMap map; - map.insert("Sampling format", QString("meta.media.%1.codec.sample_fmt").arg(audioStream)); - map.insert("Sampling rate", QString("meta.media.%1.codec.sample_rate").arg(audioStream)); - map.insert("Bit rate", QString("meta.media.%1.codec.bit_rate").arg(audioStream)); - map.insert("Channels", QString("meta.media.%1.codec.channels").arg(audioStream)); - map.insert("Codec", QString("meta.media.%1.codec.name").arg(audioStream)); - map.insert("Codec, long name", QString("meta.media.%1.codec.long_name").arg(audioStream)); - - std::cout << "Audio properties (stream " << audioStream << ")" << std::endl; - foreach (QString key, map.keys()) { - std::string value = map.value(key).toStdString(); - std::cout << "\t" << key.toStdString() << ": " << producer.get(value.c_str()) - << " (" << value << ")" << std::endl; - } - } - - return 0; -} +#include "audioInfo.h" +#include "audioStreamInfo.h" int main(int argc, char *argv[]) { @@ -73,60 +37,90 @@ int main(int argc, char *argv[]) std::cout << "Trying to align (1)\n\t" << fileSub << "\nto fit on (2)\n\t" << fileMain << "\n, result will indicate by how much (1) has to be moved." << std::endl; + // Initialize MLT + Mlt::Factory::init(NULL); + + // Load an arbitrary profile + Mlt::Profile prof("hdv_1080_25p"); - info(fileMain); - info(fileSub); + // Load the MLT producers + Mlt::Producer prodMain(prof, fileMain); + if (!prodMain.is_valid()) { + std::cout << fileMain << " is invalid." << std::endl; + return 2; + } + Mlt::Producer profSub(prof, fileSub); + if (!profSub.is_valid()) { + std::cout << fileSub << " is invalid." << std::endl; + return 2; + } -/* - Mlt::Frame *frame = producer.get_frame(); + AudioInfo infoMain(&prodMain); + AudioInfo infoSub(&profSub); + infoMain.dumpInfo(); + infoSub.dumpInfo(); + prodMain.get_fps(); - float *data = (float*)frame->get_audio(format, samplingRate, channels, nSamples); - producer.set("video_index", "-1"); - if (KdenliveSettings::normaliseaudiothumbs()) { - Mlt::Filter m_convert(prof, "volume"); - m_convert.set("gain", "normalise"); - producer.attach(m_convert); + int framesToFetch = prodMain.get_length(); + std::cout << "Length: " << framesToFetch + << " (Seconds: " << framesToFetch/prodMain.get_fps() << ")" + << std::endl; + if (framesToFetch > 5000) { + framesToFetch = 5000; } - int last_val = 0; - int val = 0; - double framesPerSecond = mlt_producer_get_fps(producer.get_producer()); - Mlt::Frame *mlt_frame; + mlt_audio_format format_s16 = mlt_audio_s16; + int samplingRate = infoMain.info(0)->samplingRate(); + int channels = 1; - for (int z = (int) frame; z < (int)(frame + lengthInFrames) && producer.is_valid() && !m_abortAudioThumb; z++) { - val = (int)((z - frame) / (frame + lengthInFrames) * 100.0); - if (last_val != val && val > 1) { - setThumbsProgress(i18n("Creating audio thumbnail for %1", url.fileName()), val); - last_val = val; + Mlt::Frame *frame; + int64_t position; + int samples; + + uint64_t envelope[framesToFetch]; + uint64_t max = 0; + + QTime t; + t.start(); + for (int i = 0; i < framesToFetch; i++) { + + frame = prodMain.get_frame(i); + position = mlt_frame_get_position(frame->get_frame()); + samples = mlt_sample_calculator(prodMain.get_fps(), infoMain.info(0)->samplingRate(), position); + + int16_t *data = static_cast(frame->get_audio(format_s16, samplingRate, channels, samples)); + + uint64_t sum = 0; + for (int k = 0; k < samples; k++) { + sum += fabs(data[k]); } - producer.seek(z); - mlt_frame = producer.get_frame(); - if (mlt_frame && mlt_frame->is_valid()) { - int samples = mlt_sample_calculator(framesPerSecond, frequency, mlt_frame_get_position(mlt_frame->get_frame())); - qint16* pcm = static_cast(mlt_frame->get_audio(audioFormat, frequency, channels, samples)); - for (int c = 0; c < channels; c++) { - QByteArray audioArray; - audioArray.resize(arrayWidth); - for (int i = 0; i < audioArray.size(); i++) { - audioArray[i] = ((*(pcm + c + i * samples / audioArray.size())) >> 9) + 127 / 2 ; - } - f.write(audioArray); - storeIn[z][c] = audioArray; - } - } else { - f.write(QByteArray(arrayWidth, '\x00')); + envelope[i] = sum; + + if (sum > max) { + max = sum; } - delete mlt_frame; } - f.close(); - setThumbsProgress(i18n("Creating audio thumbnail for %1", url.fileName()), -1); - if (m_abortAudioThumb) { - f.remove(); - } else { - clip->updateAudioThumbnail(storeIn); - }*/ + std::cout << "Calculating the envelope (" << framesToFetch << " frames) took " + << t.elapsed() << " ms." << std::endl; + + QImage img(framesToFetch, 400, QImage::Format_ARGB32); + img.fill(qRgb(255,255,255)); + double fy; + for (int x = 0; x < img.width(); x++) { + fy = envelope[x]/double(max) * img.height(); + for (int y = img.height()-1; y > img.height()-1-fy; y--) { + img.setPixel(x,y, qRgb(50, 50, 50)); + } + } + + QString outImg = QString("envelope-%1.png") + .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd-hh:mm:ss")); + img.save(outImg); + std::cout << "Saved volume envelope as " + << QFileInfo(outImg).absoluteFilePath().toStdString() + << std::endl; return 0; diff --git a/testingArea/audioStreamInfo.cpp b/testingArea/audioStreamInfo.cpp new file mode 100644 index 00000000..0c27822c --- /dev/null +++ b/testingArea/audioStreamInfo.cpp @@ -0,0 +1,66 @@ +/*************************************************************************** + * Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com) * + * This file is part of kdenlive. See www.kdenlive.org. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + ***************************************************************************/ + +#include "audioStreamInfo.h" + +#include +#include + +AudioStreamInfo::AudioStreamInfo(Mlt::Producer *producer, int audioStreamIndex) : + m_audioStreamIndex(audioStreamIndex) +{ + + std::string key; + + key = QString("meta.media.%1.codec.sample_fmt").arg(audioStreamIndex).toStdString(); + m_samplingFormat = QString(producer->get(key.c_str())); + + key = QString("meta.media.%1.codec.sample_rate").arg(audioStreamIndex).toStdString(); + m_samplingRate = atoi(producer->get(key.c_str())); + + key = QString("meta.media.%1.codec.bit_rate").arg(audioStreamIndex).toStdString(); + m_bitRate = atoi(producer->get(key.c_str())); + + key = QString("meta.media.%1.codec.channels").arg(audioStreamIndex).toStdString(); + m_channels = atoi(producer->get(key.c_str())); + + key = QString("meta.media.%1.codec.name").arg(audioStreamIndex).toStdString(); + m_codecName = QString(producer->get(key.c_str())); + + key = QString("meta.media.%1.codec.long_name").arg(audioStreamIndex).toStdString(); + m_codecLongName = QString(producer->get(key.c_str())); +} +AudioStreamInfo::~AudioStreamInfo() +{ +} + +int AudioStreamInfo::streamIndex() const { return m_audioStreamIndex; } +int AudioStreamInfo::samplingRate() const { return m_samplingRate; } +int AudioStreamInfo::channels() const { return m_channels; } +int AudioStreamInfo::bitrate() const { return m_bitRate; } +const QString& AudioStreamInfo::codecName(bool longName) const +{ + if (longName) { + return m_codecLongName; + } else { + return m_codecName; + } +} + +void AudioStreamInfo::dumpInfo() const +{ + std::cout << "Info for audio stream " << m_audioStreamIndex << std::endl + << "\tCodec: " << m_codecLongName.toStdString() << " (" << m_codecName.toStdString() << ")" << std::endl + << "\tChannels: " << m_channels << std::endl + << "\tSampling rate: " << m_samplingRate << std::endl + << "\tBit rate: " << m_bitRate << std::endl + ; + +} diff --git a/testingArea/audioStreamInfo.h b/testingArea/audioStreamInfo.h new file mode 100644 index 00000000..92d9e434 --- /dev/null +++ b/testingArea/audioStreamInfo.h @@ -0,0 +1,44 @@ +/*************************************************************************** + * Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com) * + * This file is part of kdenlive. See www.kdenlive.org. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + ***************************************************************************/ + +#ifndef AUDIOSTREAMINFO_H +#define AUDIOSTREAMINFO_H + +#include +#include + +class AudioStreamInfo +{ +public: + AudioStreamInfo(Mlt::Producer *producer, int audioStreamIndex); + ~AudioStreamInfo(); + + int streamIndex() const; + int samplingRate() const; + int channels() const; + int bitrate() const; + const QString& codecName(bool longName = false) const; + const QString& samplingFormat() const; + + void dumpInfo() const; + +private: + int m_audioStreamIndex; + + int m_samplingRate; + int m_channels; + int m_bitRate; + QString m_codecName; + QString m_codecLongName; + QString m_samplingFormat; + +}; + +#endif // AUDIOSTREAMINFO_H -- 2.39.5