1 /***************************************************************************
2 mltdevicecapture.cpp - description
4 begin : Sun May 21 2011
5 copyright : (C) 2011 by Jean-Baptiste Mardelle (jb@kdenlive.org)
7 ***************************************************************************/
9 /***************************************************************************
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
16 ***************************************************************************/
19 #include "mltdevicecapture.h"
20 #include "kdenlivesettings.h"
21 #include "definitions.h"
22 //#include "recmonitor.h"
23 //#include "renderer.h"
24 #include "blackmagic/devices.h"
26 #include <mlt++/Mlt.h>
29 #include <KStandardDirs>
30 #include <KMessageBox>
32 #include <KTemporaryFile>
37 #include <QApplication>
47 static void consumer_gl_frame_show(mlt_consumer, MltDeviceCapture * self, mlt_frame frame_ptr)
49 // detect if the producer has finished playing. Is there a better way to do it?
50 Mlt::Frame frame(frame_ptr);
51 self->showFrame(frame);
54 static void rec_consumer_frame_show(mlt_consumer, MltDeviceCapture * self, mlt_frame frame_ptr)
56 Mlt::Frame frame(frame_ptr);
57 if (!frame.is_valid()) return;
58 self->gotCapturedFrame(frame);
61 static void rec_consumer_frame_preview(mlt_consumer, MltDeviceCapture * self, mlt_frame frame_ptr)
63 Mlt::Frame frame(frame_ptr);
64 if (!frame.is_valid()) return;
65 if (self->sendFrameForAnalysis && frame_ptr->convert_image) {
66 self->emitFrameUpdated(frame);
68 if (self->doCapture > 0) {
70 if (self->doCapture == 0) self->saveFrame(frame);
73 //TODO: connect record monitor to audio scopes
75 if (self->analyseAudio) {
76 self->showAudio(frame);
82 MltDeviceCapture::MltDeviceCapture(QString profile, VideoPreviewContainer *surface, QWidget *parent) :
83 AbstractRender("capture", parent),
85 sendFrameForAnalysis(false),
90 m_livePreview(KdenliveSettings::recording_preview()),
91 m_captureDisplayWidget(surface),
92 m_winid((int) surface->winId()),
93 analyseAudio(KdenliveSettings::monitor_audio()),
94 processingImage(false)
96 if (profile.isEmpty()) profile = KdenliveSettings::current_profile();
97 buildConsumer(profile);
98 connect(this, SIGNAL(unblockPreview()), this, SLOT(slotPreparePreview()));
101 MltDeviceCapture::~MltDeviceCapture()
103 if (m_mltConsumer) delete m_mltConsumer;
104 if (m_mltProducer) delete m_mltProducer;
105 if (m_mltProfile) delete m_mltProfile;
108 void MltDeviceCapture::buildConsumer(const QString &profileName)
110 if (!profileName.isEmpty()) m_activeProfile = profileName;
112 if (m_mltProfile) delete m_mltProfile;
114 char *tmp = qstrdup(m_activeProfile.toUtf8().constData());
115 setenv("MLT_PROFILE", tmp, 1);
116 m_mltProfile = new Mlt::Profile(tmp);
117 m_mltProfile->get_profile()->is_explicit = 1;
120 QString videoDriver = KdenliveSettings::videodrivername();
121 if (!videoDriver.isEmpty()) {
122 if (videoDriver == "x11_noaccel") {
123 setenv("SDL_VIDEO_YUV_HWACCEL", "0", 1);
126 unsetenv("SDL_VIDEO_YUV_HWACCEL");
129 setenv("SDL_VIDEO_ALLOW_SCREENSAVER", "1", 1);
133 m_mltConsumer = new Mlt::Consumer(*m_mltProfile, "sdl_audio");
134 m_mltConsumer->set("preview_off", 1);
135 m_mltConsumer->set("preview_format", mlt_image_rgb24);
136 m_mltConsumer->listen("consumer-frame-show", this, (mlt_listener) consumer_gl_frame_show);
138 m_mltConsumer = new Mlt::Consumer(*m_mltProfile, "sdl_preview");
139 m_mltConsumer->set("window_id", m_winid);
140 m_mltConsumer->listen("consumer-frame-show", this, (mlt_listener) rec_consumer_frame_preview);
142 //m_mltConsumer->set("resize", 1);
143 //m_mltConsumer->set("terminate_on_pause", 1);
144 m_mltConsumer->set("window_background", KdenliveSettings::window_background().name().toUtf8().constData());
145 //m_mltConsumer->set("rescale", "nearest");
147 QString audioDevice = KdenliveSettings::audiodevicename();
148 if (!audioDevice.isEmpty())
149 m_mltConsumer->set("audio_device", audioDevice.toUtf8().constData());
151 if (!videoDriver.isEmpty())
152 m_mltConsumer->set("video_driver", videoDriver.toUtf8().constData());
154 QString audioDriver = KdenliveSettings::audiodrivername();
156 if (!audioDriver.isEmpty())
157 m_mltConsumer->set("audio_driver", audioDriver.toUtf8().constData());
159 //m_mltConsumer->set("progressive", 0);
160 //m_mltConsumer->set("buffer", 1);
161 //m_mltConsumer->set("real_time", 0);
164 void MltDeviceCapture::stop()
166 bool isPlaylist = false;
167 disconnect(this, SIGNAL(imageReady(QImage)), this, SIGNAL(frameUpdated(QImage)));
168 m_captureDisplayWidget->stop();
170 m_mltConsumer->set("refresh", 0);
171 m_mltConsumer->stop();
172 //if (!m_mltConsumer->is_stopped()) m_mltConsumer->stop();
175 QList <Mlt::Producer *> prods;
176 Mlt::Service service(m_mltProducer->parent().get_service());
177 mlt_service_lock(service.get_service());
178 if (service.type() == tractor_type) {
180 Mlt::Tractor tractor(service);
181 mlt_tractor_close(tractor.get_tractor());
182 Mlt::Field *field = tractor.field();
183 mlt_service nextservice = mlt_service_get_producer(service.get_service());
184 mlt_service nextservicetodisconnect;
185 mlt_properties properties = MLT_SERVICE_PROPERTIES(nextservice);
186 QString mlt_type = mlt_properties_get(properties, "mlt_type");
187 QString resource = mlt_properties_get(properties, "mlt_service");
188 // Delete all transitions
189 while (mlt_type == "transition") {
190 nextservicetodisconnect = nextservice;
191 nextservice = mlt_service_producer(nextservice);
192 mlt_field_disconnect_service(field->get_field(), nextservicetodisconnect);
193 if (nextservice == NULL) break;
194 properties = MLT_SERVICE_PROPERTIES(nextservice);
195 mlt_type = mlt_properties_get(properties, "mlt_type");
196 resource = mlt_properties_get(properties, "mlt_service");
198 for (int trackNb = tractor.count() - 1; trackNb >= 0; --trackNb) {
199 Mlt::Producer trackProducer(tractor.track(trackNb));
200 Mlt::Playlist trackPlaylist((mlt_playlist) trackProducer.get_service());
201 if (trackPlaylist.type() == playlist_type) {
202 for (int i = 0; i < trackPlaylist.count();i++) {
203 // We need to manually decrease the ref count and close the producer, otherwise
204 // the video4linux device stays open, seems like a bug in MLT that is not cleaning properly
205 mlt_properties props = MLT_PRODUCER_PROPERTIES(trackPlaylist.get_clip(i)->get_parent());
206 while (mlt_properties_ref_count(props) > 0) mlt_properties_dec_ref(props);
207 if (trackPlaylist.get_clip(i)) mlt_producer_close(trackPlaylist.get_clip(i)->get_parent());
209 mlt_playlist_close(trackPlaylist.get_playlist());
215 mlt_service_unlock(service.get_service());
216 delete m_mltProducer;
217 m_mltProducer = NULL;
219 // For some reason, the consumer seems to be deleted by previous stuff when in playlist mode
220 if (!isPlaylist && m_mltConsumer) delete m_mltConsumer;
221 m_mltConsumer = NULL;
225 void MltDeviceCapture::doRefresh()
227 if (m_mltConsumer) m_mltConsumer->set("refresh", 1);
231 void MltDeviceCapture::emitFrameUpdated(Mlt::Frame& frame)
234 //TEST: is it better to convert the frame in a thread outside of MLT??
235 if (processingImage) return;
236 mlt_image_format format = (mlt_image_format) frame.get_int("format"); //mlt_image_rgb24;
237 int width = frame.get_int("width");
238 int height = frame.get_int("height");
239 unsigned char *buffer = (unsigned char *) frame.get_data("image");
240 if (format == mlt_image_yuv422) {
241 QtConcurrent::run(this, &MltDeviceCapture::uyvy2rgb, (unsigned char *) buffer, width, height);
245 mlt_image_format format = mlt_image_rgb24;
248 const uchar* image = frame.get_image(format, width, height);
249 QImage qimage(width, height, QImage::Format_ARGB32);
250 memcpy(qimage.bits(), image, width * height * 3);
251 emit frameUpdated(qimage.rgbSwapped());
254 void MltDeviceCapture::showFrame(Mlt::Frame& frame)
256 mlt_image_format format = mlt_image_rgb24;
259 const uchar* image = frame.get_image(format, width, height);
260 QImage qimage(width, height, QImage::Format_RGB888);
261 memcpy(qimage.scanLine(0), image, width * height * 3);
262 emit showImageSignal(qimage);
264 if (sendFrameForAnalysis && frame.get_frame()->convert_image) {
265 emit frameUpdated(qimage.rgbSwapped());
269 void MltDeviceCapture::showAudio(Mlt::Frame& frame)
271 if (!frame.is_valid() || frame.get_int("test_audio") != 0) {
274 mlt_audio_format audio_format = mlt_audio_s16;
276 int num_channels = 0;
278 int16_t* data = (int16_t*)frame.get_audio(audio_format, freq, num_channels, samples);
284 // Data format: [ c00 c10 c01 c11 c02 c12 c03 c13 ... c0{samples-1} c1{samples-1} for 2 channels.
285 // So the vector is of size samples*channels.
286 QVector<int16_t> sampleVector(samples*num_channels);
287 memcpy(sampleVector.data(), data, samples*num_channels*sizeof(int16_t));
290 emit audioSamplesSignal(sampleVector, freq, num_channels, samples);
294 bool MltDeviceCapture::slotStartPreview(const QString &producer, bool xmlFormat)
296 if (m_mltConsumer == NULL) {
299 char *tmp = qstrdup(producer.toUtf8().constData());
300 if (xmlFormat) m_mltProducer = new Mlt::Producer(*m_mltProfile, "xml-string", tmp);
301 else m_mltProducer = new Mlt::Producer(*m_mltProfile, tmp);
304 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) {
306 delete m_mltProducer;
307 m_mltProducer = NULL;
309 kDebug()<<"//// ERROR CREATRING PROD";
312 m_mltConsumer->connect(*m_mltProducer);
313 if (m_mltConsumer->start() == -1) {
314 delete m_mltConsumer;
315 m_mltConsumer = NULL;
318 connect(this, SIGNAL(imageReady(QImage)), this, SIGNAL(frameUpdated(QImage)));
322 void MltDeviceCapture::gotCapturedFrame(Mlt::Frame& frame)
325 int dropped = m_mltProducer->get_int("dropped");
326 if (dropped > m_droppedFrames) {
327 m_droppedFrames = dropped;
328 emit droppedFrames(m_droppedFrames);
332 if (m_livePreview == 2) return;
333 if (m_livePreview == 0 && (m_frameCount % 10 > 0)) return;
334 mlt_image_format format = mlt_image_rgb24;
337 uint8_t *data = frame.get_image(format, width, height, 0);
338 //QImage image(width, height, QImage::Format_RGB888);
339 //memcpy(image.bits(), data, width * height * 3);
340 QImage image((uchar *)data, width, height, QImage::Format_RGB888);
342 m_captureDisplayWidget->setImage(image);
344 //TEST: is it better to process frame conversion ouside MLT???
346 if (!m_livePreview || processingImage) return;
348 mlt_image_format format = (mlt_image_format) frame.get_int("format"); //mlt_image_rgb24a;
349 int width = frame.get_int("width");
350 int height = frame.get_int("height");
351 unsigned char *buffer = (unsigned char *) frame.get_data("image");
352 //unsigned char *buffer = frame.get_image(format, width, height);
353 //convert from uyvy422 to rgba
354 if (format == mlt_image_yuv422) {
355 QtConcurrent::run(this, &MltDeviceCapture::uyvy2rgb, (unsigned char *) buffer, width, height);
356 //CaptureHandler::uyvy2rgb((uchar *)frameBytes, (uchar *)image.bits(), videoFrame->GetWidth(), videoFrame->GetHeight());
360 void MltDeviceCapture::saveFrame(Mlt::Frame& frame)
362 mlt_image_format format = mlt_image_rgb24;
365 const uchar* image = frame.get_image(format, width, height);
366 QImage qimage(width, height, QImage::Format_RGB888);
367 memcpy(qimage.bits(), image, width * height * 3);
370 Mlt::Service service(m_mltProducer->parent().get_service());
371 Mlt::Tractor tractor(service);
372 Mlt::Producer trackProducer(tractor.track(0));
373 trackProducer.set("hide", 0);
375 qimage.rgbSwapped().save(m_capturePath);
376 emit frameSaved(m_capturePath);
377 m_capturePath.clear();
380 void MltDeviceCapture::captureFrame(const QString &path)
382 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) return;
384 // Hide overlay track before doing the capture
385 Mlt::Service service(m_mltProducer->parent().get_service());
386 Mlt::Tractor tractor(service);
387 Mlt::Producer trackProducer(tractor.track(0));
388 mlt_service_lock(service.get_service());
389 trackProducer.set("hide", 1);
390 m_mltConsumer->purge();
391 mlt_service_unlock(service.get_service());
392 m_capturePath = path;
393 // Wait for 5 frames before capture to make sure overlay is gone
397 bool MltDeviceCapture::slotStartCapture(const QString ¶ms, const QString &path, const QString &playlist, int livePreview, bool xmlPlaylist)
400 m_livePreview = livePreview;
403 if (m_mltProfile) delete m_mltProfile;
404 char *tmp = qstrdup(m_activeProfile.toUtf8().constData());
405 m_mltProfile = new Mlt::Profile(tmp);
407 m_mltProfile->get_profile()->is_explicit = 1;
408 kDebug()<<"-- CREATING CAP: "<<params<<", PATH: "<<path;
409 tmp = qstrdup(QString("avformat:" + path).toUtf8().constData());
410 m_mltConsumer = new Mlt::Consumer(*m_mltProfile, tmp);
411 m_mltConsumer->set("real_time", KdenliveSettings::mltthreads());
414 QStringList paramList = params.split(" ", QString::SkipEmptyParts);
416 for (int i = 0; i < paramList.count(); i++) {
417 tmp = qstrdup(paramList.at(i).section("=", 0, 0).toUtf8().constData());
418 QString value = paramList.at(i).section("=", 1, 1);
419 if (value == "%threads") value = QString::number(QThread::idealThreadCount());
420 tmp2 = qstrdup(value.toUtf8().constData());
421 m_mltConsumer->set(tmp, tmp2);
426 if (m_mltConsumer == NULL || !m_mltConsumer->is_valid()) {
428 delete m_mltConsumer;
429 m_mltConsumer = NULL;
434 // FIXME: the event object returned by the listen gets leaked...
435 m_mltConsumer->listen("consumer-frame-render", this, (mlt_listener) rec_consumer_frame_show);
436 tmp = qstrdup(playlist.toUtf8().constData());
438 // create an xml producer
439 m_mltProducer = new Mlt::Producer(*m_mltProfile, "xml-string", tmp);
442 // create a producer based on mltproducer parameter
443 m_mltProducer = new Mlt::Producer(*m_mltProfile, tmp);
447 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) {
448 kDebug()<<"//// ERROR CREATRING PROD";
452 m_mltConsumer->connect(*m_mltProducer);
453 if (m_mltConsumer->start() == -1) {
454 delete m_mltConsumer;
455 m_mltConsumer = NULL;
458 m_captureDisplayWidget->start();
463 void MltDeviceCapture::setOverlay(const QString &path)
465 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) return;
466 Mlt::Producer parentProd(m_mltProducer->parent());
467 if (parentProd.get_producer() == NULL) {
468 kDebug() << "PLAYLIST BROKEN, CANNOT INSERT CLIP //////";
472 Mlt::Service service(parentProd.get_service());
473 if (service.type() != tractor_type) {
474 kWarning() << "// TRACTOR PROBLEM";
477 Mlt::Tractor tractor(service);
478 if ( tractor.count() < 2) {
479 kWarning() << "// TRACTOR PROBLEM";
482 mlt_service_lock(service.get_service());
483 Mlt::Producer trackProducer(tractor.track(0));
484 Mlt::Playlist trackPlaylist((mlt_playlist) trackProducer.get_service());
486 trackPlaylist.remove(0);
487 if (path.isEmpty()) {
488 mlt_service_unlock(service.get_service());
493 char *tmp = qstrdup(path.toUtf8().constData());
494 Mlt::Producer *clip = new Mlt::Producer (*m_mltProfile, "loader", tmp);
496 clip->set_in_and_out(0, 99999);
497 trackPlaylist.insert_at(0, clip, 1);
500 mlt_service serv = m_mltProducer->parent().get_service();
501 mlt_service nextservice = mlt_service_get_producer(serv);
502 mlt_properties properties = MLT_SERVICE_PROPERTIES(nextservice);
503 QString mlt_type = mlt_properties_get(properties, "mlt_type");
504 if (mlt_type != "transition") {
505 // transition does not exist, add it
506 Mlt::Field *field = tractor.field();
507 Mlt::Transition *transition = new Mlt::Transition(*m_mltProfile, "composite");
508 transition->set_in_and_out(0, 0);
509 transition->set("geometry", "0,0:100%x100%:70");
510 transition->set("fill", 1);
511 transition->set("operator", "and");
512 transition->set("a_track", 0);
513 transition->set("b_track", 1);
514 field->plant_transition(*transition, 0, 1);
516 mlt_service_unlock(service.get_service());
520 void MltDeviceCapture::setOverlayEffect(const QString tag, QStringList parameters)
522 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) return;
523 Mlt::Service service(m_mltProducer->parent().get_service());
524 Mlt::Tractor tractor(service);
525 Mlt::Producer trackProducer(tractor.track(0));
526 Mlt::Playlist trackPlaylist((mlt_playlist) trackProducer.get_service());
527 Mlt::Service trackService(trackProducer.get_service());
529 mlt_service_lock(service.get_service());
531 // delete previous effects
533 filter = trackService.filter(0);
534 if (filter && !tag.isEmpty()) {
535 QString currentService = filter->get("mlt_service");
536 if (currentService == tag) {
537 // Effect is already there
538 mlt_service_unlock(service.get_service());
543 trackService.detach(*filter);
545 filter = trackService.filter(0);
549 mlt_service_unlock(service.get_service());
553 char *tmp = qstrdup(tag.toUtf8().constData());
554 filter = new Mlt::Filter(*m_mltProfile, tmp);
556 if (filter && filter->is_valid()) {
557 for (int j = 0; j < parameters.count(); j++) {
558 filter->set(parameters.at(j).section("=", 0, 0).toUtf8().constData(), parameters.at(j).section("=", 1, 1).toUtf8().constData());
560 trackService.attach(*filter);
562 mlt_service_unlock(service.get_service());
565 void MltDeviceCapture::mirror(bool activate)
567 if (m_mltProducer == NULL || !m_mltProducer->is_valid()) return;
568 Mlt::Service service(m_mltProducer->parent().get_service());
569 Mlt::Tractor tractor(service);
570 Mlt::Producer trackProducer(tractor.track(1));
571 Mlt::Playlist trackPlaylist((mlt_playlist) trackProducer.get_service());
572 Mlt::Service trackService(trackProducer.get_service());
574 mlt_service_lock(service.get_service());
576 // delete previous effects
578 filter = trackService.filter(0);
580 trackService.detach(*filter);
582 filter = trackService.filter(0);
586 mlt_service_unlock(service.get_service());
590 filter = new Mlt::Filter(*m_mltProfile, "mirror");
591 if (filter && filter->is_valid()) {
592 filter->set("mirror", "flip");
593 trackService.attach(*filter);
595 mlt_service_unlock(service.get_service());
598 void MltDeviceCapture::uyvy2rgb(unsigned char *yuv_buffer, int width, int height)
600 processingImage = true;
601 QImage image(width, height, QImage::Format_RGB888);
602 unsigned char *rgb_buffer = image.bits();
607 int rgb_ptr, y_ptr, t;
609 len = width * height / 2;
614 for (t = 0; t < len; t++) {
617 Y = yuv_buffer[y_ptr];
618 U = yuv_buffer[y_ptr+1];
619 Y2 = yuv_buffer[y_ptr+2];
620 V = yuv_buffer[y_ptr+3];
623 r = ((298 * (Y - 16) + 409 * (V - 128) + 128) >> 8);
625 g = ((298 * (Y - 16) - 100 * (U - 128) - 208 * (V - 128) + 128) >> 8);
627 b = ((298 * (Y - 16) + 516 * (U - 128) + 128) >> 8);
629 if (r > 255) r = 255;
630 if (g > 255) g = 255;
631 if (b > 255) b = 255;
637 rgb_buffer[rgb_ptr] = r;
638 rgb_buffer[rgb_ptr+1] = g;
639 rgb_buffer[rgb_ptr+2] = b;
643 r = ((298 * (Y2 - 16) + 409 * (V - 128) + 128) >> 8);
645 g = ((298 * (Y2 - 16) - 100 * (U - 128) - 208 * (V - 128) + 128) >> 8);
647 b = ((298 * (Y2 - 16) + 516 * (U - 128) + 128) >> 8);
649 if (r > 255) r = 255;
650 if (g > 255) g = 255;
651 if (b > 255) b = 255;
657 rgb_buffer[rgb_ptr] = r;
658 rgb_buffer[rgb_ptr+1] = g;
659 rgb_buffer[rgb_ptr+2] = b;
662 //emit imageReady(image);
663 m_captureDisplayWidget->setImage(image);
664 emit unblockPreview();
665 //processingImage = false;
668 void MltDeviceCapture::slotPreparePreview()
670 QTimer::singleShot(1000, this, SLOT(slotAllowPreview()));
673 void MltDeviceCapture::slotAllowPreview()
675 processingImage = false;