1 /*****************************************************************************
2 * qtsound.m: qtkit (Mac OS X) based audio capture module
3 *****************************************************************************
4 * Copyright © 2011 VLC authors and VideoLAN
6 * Authors: Pierre d'Herbemont <pdherbemont@videolan.org>
7 * Gustaf Neumann <neumann@wu.ac.at>
8 * Michael S. Feurstein <michael.feurstein@wu.ac.at>
10 *****************************************************************************
11 * This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * This library is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with this library; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
25 *****************************************************************************/
27 /*****************************************************************************
29 *****************************************************************************/
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
39 #include <vlc_demux.h>
40 #include <vlc_dialog.h>
42 //#define QTKIT_VERSION_MIN_REQUIRED 70603
43 #define QTKIT_VERSION_MAX_ALLOWED 70700
45 #import <QTKit/QTKit.h>
47 /*****************************************************************************
49 *****************************************************************************/
50 static int Open(vlc_object_t *p_this);
51 static void Close(vlc_object_t *p_this);
52 static int Demux(demux_t *p_demux);
53 static int Control(demux_t *, int, va_list);
55 /*****************************************************************************
57 *****************************************************************************/
60 set_shortname(N_("QTSound"))
61 set_description(N_("QuickTime Sound Capture"))
62 set_category(CAT_INPUT)
63 set_subcategory(SUBCAT_INPUT_ACCESS)
64 add_shortcut("qtsound")
65 set_capability("access_demux", 0)
66 set_callbacks(Open, Close)
70 /*****************************************************************************
72 *****************************************************************************/
73 @interface VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
76 AudioBuffer *currentAudioBuffer;
77 block_t *rawAudioData;
78 UInt32 numberOfSamples;
83 - (id)initWithDemux:(demux_t *)p_demux;
84 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
85 - (BOOL)checkCurrentAudioBuffer;
87 - (mtime_t)getCurrentPts;
88 - (void *)getCurrentAudioBufferData;
89 - (UInt32)getCurrentTotalDataSize;
90 - (UInt32)getNumberOfSamples;
94 @implementation VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
95 - (id)initWithDemux:(demux_t *)p_demux
97 if (self = [super init]) {
99 currentAudioBuffer = nil;
100 date_Init(&date, 44100, 1);
112 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection
114 AudioBufferList *tempAudioBufferList;
115 UInt32 totalDataSize = 0;
118 @synchronized (self) {
119 numberOfSamples = [sampleBuffer numberOfSamples];
120 date_Increment(&date,numberOfSamples);
121 currentPts = date_Get(&date);
123 tempAudioBufferList = [sampleBuffer audioBufferListWithOptions:0];
124 if (tempAudioBufferList->mNumberBuffers == 2) {
126 * Compute totalDataSize as sum of all data blocks in the
129 for (count = 0; count < tempAudioBufferList->mNumberBuffers; count++)
130 totalDataSize += tempAudioBufferList->mBuffers[count].mDataByteSize;
133 * Allocate storage for the interleaved audio data
135 rawAudioData = block_Alloc(totalDataSize * sizeof(float));
136 if (NULL == rawAudioData) {
137 msg_Err(p_qtsound, "Raw audiodata could not be allocated");
141 msg_Err(p_qtsound, "Too many or only one channel found: %i.",
142 tempAudioBufferList->mNumberBuffers);
147 * Interleave raw data (provided in two separate channels as
148 * F32L) with 2 samples per frame
152 const float *b1Ptr, *b2Ptr;
156 uPtr = (float *)rawAudioData,
157 b1Ptr = (const float *) tempAudioBufferList->mBuffers[0].mData,
158 b2Ptr = (const float *) tempAudioBufferList->mBuffers[1].mData;
159 i < numberOfSamples; i++) {
164 if (currentAudioBuffer == nil) {
165 currentAudioBuffer = (AudioBuffer *)malloc(sizeof(AudioBuffer));
166 if (NULL == currentAudioBuffer) {
167 msg_Err(p_qtsound, "AudioBuffer could not be allocated.");
171 currentAudioBuffer->mNumberChannels = 2;
172 currentAudioBuffer->mDataByteSize = totalDataSize;
173 currentAudioBuffer->mData = rawAudioData;
178 - (BOOL)checkCurrentAudioBuffer
180 return (currentAudioBuffer) ? 1 : 0;
185 @synchronized (self) {
187 block_Release(rawAudioData);
191 - (mtime_t)getCurrentPts
193 /* FIXME: can this getter be minimized? */
196 if(!currentAudioBuffer || currentPts == previousPts)
199 @synchronized (self) {
200 pts = previousPts = currentPts;
203 return (currentAudioBuffer->mData) ? currentPts : 0;
206 - (void *)getCurrentAudioBufferData
208 return currentAudioBuffer->mData;
211 - (UInt32)getCurrentTotalDataSize
213 return currentAudioBuffer->mDataByteSize;
216 - (UInt32)getNumberOfSamples
218 return numberOfSamples;
223 /*****************************************************************************
225 *****************************************************************************/
228 QTCaptureSession * session;
229 QTCaptureDevice * audiodevice;
230 VLCDecompressedAudioOutput * audiooutput;
231 es_out_id_t *p_es_audio;
232 int i_audio_max_buffer_size;
235 /*****************************************************************************
236 * Open: initialize interface
237 *****************************************************************************/
238 static int Open(vlc_object_t *p_this)
240 demux_t *p_demux = (demux_t*)p_this;
242 es_format_t audiofmt;
243 char *psz_uid = NULL;
246 NSString *qtk_curraudiodevice_uid;
247 NSAutoreleasePool *pool;
248 NSArray *myAudioDevices, *audioformat_array;
249 QTFormatDescription *audio_format;
250 QTCaptureDeviceInput *audioInput;
251 NSError *o_returnedAudioError;
253 if(p_demux->psz_location && *p_demux->psz_location)
254 psz_uid = p_demux->psz_location;
256 msg_Dbg(p_demux, "qtsound uid = %s", psz_uid);
257 qtk_curraudiodevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
259 pool = [[NSAutoreleasePool alloc] init];
261 p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
265 msg_Dbg(p_demux, "qtsound : uid = %s", [qtk_curraudiodevice_uid UTF8String]);
266 myAudioDevices = [[[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeSound] arrayByAddingObjectsFromArray:[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeMuxed]] retain];
267 if([myAudioDevices count] == 0) {
268 dialog_FatalWait(p_demux, _("No Audio Input device found"),
269 _("Your Mac does not seem to be equipped with a suitable audio input device."
270 "Please check your connectors and drivers."));
271 msg_Err(p_demux, "Can't find any Audio device");
276 for (iaudio = 0; iaudio < [myAudioDevices count]; iaudio++) {
277 QTCaptureDevice *qtk_audioDevice;
278 qtk_audioDevice = [myAudioDevices objectAtIndex:iaudio];
279 msg_Dbg(p_demux, "qtsound audio %u/%lu localizedDisplayName: %s uniqueID: %s", iaudio, [myAudioDevices count], [[qtk_audioDevice localizedDisplayName] UTF8String], [[qtk_audioDevice uniqueID] UTF8String]);
280 if ([[[qtk_audioDevice uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:qtk_curraudiodevice_uid]) {
281 msg_Dbg(p_demux, "Device found");
287 if(iaudio < [myAudioDevices count])
288 p_sys->audiodevice = [myAudioDevices objectAtIndex:iaudio];
290 /* cannot find designated audio device, fall back to open default audio device */
291 msg_Dbg(p_demux, "Cannot find designated uid audio device as %s. Fall back to open default audio device.", [qtk_curraudiodevice_uid UTF8String]);
292 p_sys->audiodevice = [QTCaptureDevice defaultInputDeviceWithMediaType: QTMediaTypeSound];
294 if(!p_sys->audiodevice) {
295 dialog_FatalWait(p_demux, _("No audio input device found"),
296 _("Your Mac does not seem to be equipped with a suitable audio input device."
297 "Please check your connectors and drivers."));
298 msg_Err(p_demux, "Can't find any Audio device");
303 if(![p_sys->audiodevice open: &o_returnedAudioError]) {
304 msg_Err(p_demux, "Unable to open the audio capture device (%ld)", [o_returnedAudioError code]);
308 if([p_sys->audiodevice isInUseByAnotherApplication] == YES) {
309 msg_Err(p_demux, "default audio capture device is exclusively in use by another application");
312 audioInput = [[QTCaptureDeviceInput alloc] initWithDevice: p_sys->audiodevice];
314 msg_Err(p_demux, "can't create a valid audio capture input facility");
317 msg_Dbg(p_demux, "created valid audio capture input facility");
319 p_sys->audiooutput = [[VLCDecompressedAudioOutput alloc] initWithDemux:p_demux];
320 msg_Dbg (p_demux, "initialized audio output");
322 /* Get the formats */
324 FIXME: the format description gathered here does not seem to be the same
325 in comparison to the format description collected from the actual sampleBuffer.
326 This information needs to be updated some other place. For the time being this shall suffice.
328 The following verbose output is an example of what is read from the input device during the below block
329 [0x3042138] qtsound demux debug: Audio localized format summary: Linear PCM, 24 bit little-endian signed integer, 2 channels, 44100 Hz
330 [0x3042138] qtsound demux debug: Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000004; Bytes per Packet: 8; Frames per Packet: 1; Bytes per Frame: 8; Channels per Frame: 2; Bits per Channel: 24
331 [0x3042138] qtsound demux debug: Flag float 0 bigEndian 0 signedInt 1 packed 0 alignedHigh 0 non interleaved 0 non mixable 0
332 canonical 0 nativeFloatPacked 0 nativeEndian 0
334 However when reading this information from the sampleBuffer during the delegate call from
335 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
336 the following data shows up
337 2011-09-23 22:06:03.077 VLC[23070:f103] Audio localized format summary: Linear PCM, 32 bit little-endian floating point, 2 channels, 44100 Hz
338 2011-09-23 22:06:03.078 VLC[23070:f103] Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000029; Bytes per Packet: 4; Frames per Packet: 1; Bytes per Frame: 4; Channels per Frame: 2; Bits per Channel: 32
339 2011-09-23 22:06:03.078 VLC[23070:f103] Flag float 1 bigEndian 0 signedInt 0 packed 1 alignedHigh 0 non interleaved 1 non mixable 0
340 canonical 1 nativeFloatPacked 1 nativeEndian 0
344 little-endian signed integer vs. little-endian floating point
345 format flag 00000004 vs. 00000029
346 bytes per packet 8 vs. 4
348 non interleaved 0 vs. 1 -> this makes a major difference when filling our own buffer
350 nativeFloatPacked 0 vs. 1
352 One would assume we'd need to feed the (es_format_t)audiofmt with the data collected here.
353 This is not the case. Audio will be transmitted in artefacts, due to wrong information.
355 At the moment this data is set manually, however one should consider trying to set this data dynamically
357 audioformat_array = [p_sys->audiodevice formatDescriptions];
359 for(int k = 0; k < [audioformat_array count]; k++) {
360 audio_format = (QTFormatDescription *)[audioformat_array objectAtIndex:k];
362 msg_Dbg(p_demux, "Audio localized format summary: %s", [[audio_format localizedFormatSummary] UTF8String]);
363 msg_Dbg(p_demux, "Audio format description attributes: %s",[[[audio_format formatDescriptionAttributes] description] UTF8String]);
365 AudioStreamBasicDescription asbd = {0};
366 NSValue *asbdValue = [audio_format attributeForKey:QTFormatDescriptionAudioStreamBasicDescriptionAttribute];
367 [asbdValue getValue:&asbd];
369 char formatIDString[5];
370 UInt32 formatID = CFSwapInt32HostToBig (asbd.mFormatID);
371 bcopy (&formatID, formatIDString, 4);
372 formatIDString[4] = '\0';
374 /* kept for development purposes */
376 msg_Dbg(p_demux, "Sample Rate: %.0lf; Format ID: %s; Format Flags: %.8x; Bytes per Packet: %d; Frames per Packet: %d; Bytes per Frame: %d; Channels per Frame: %d; Bits per Channel: %d",
380 asbd.mBytesPerPacket,
381 asbd.mFramesPerPacket,
383 asbd.mChannelsPerFrame,
384 asbd.mBitsPerChannel);
386 msg_Dbg(p_demux, "Flag float %d bigEndian %d signedInt %d packed %d alignedHigh %d non interleaved %d non mixable %d\ncanonical %d nativeFloatPacked %d nativeEndian %d",
387 (asbd.mFormatFlags & kAudioFormatFlagIsFloat) != 0,
388 (asbd.mFormatFlags & kAudioFormatFlagIsBigEndian) != 0,
389 (asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0,
390 (asbd.mFormatFlags & kAudioFormatFlagIsPacked) != 0,
391 (asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh) != 0,
392 (asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0,
393 (asbd.mFormatFlags & kAudioFormatFlagIsNonMixable) != 0,
395 (asbd.mFormatFlags & kAudioFormatFlagsCanonical) != 0,
396 (asbd.mFormatFlags & kAudioFormatFlagsNativeFloatPacked) != 0,
397 (asbd.mFormatFlags & kAudioFormatFlagsNativeEndian) != 0
402 if([audioformat_array count])
403 audio_format = [audioformat_array objectAtIndex:0];
407 /* Now we can init */
408 audiocodec = VLC_CODEC_FL32;
409 es_format_Init(&audiofmt, AUDIO_ES, audiocodec);
411 audiofmt.audio.i_format = audiocodec;
412 audiofmt.audio.i_rate = 44100;
414 * i_physical_channels Describes the channels configuration of the
415 * samples (ie. number of channels which are available in the
416 * buffer, and positions).
418 audiofmt.audio.i_physical_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
420 * i_original_channels Describes from which original channels,
421 * before downmixing, the buffer is derived.
423 audiofmt.audio.i_original_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
425 * Please note that it may be completely arbitrary - buffers are not
426 * obliged to contain a integral number of so-called "frames". It's
427 * just here for the division:
428 * buffer_size = i_nb_samples * i_bytes_per_frame / i_frame_length
430 audiofmt.audio.i_bitspersample = 32;
431 audiofmt.audio.i_channels = 2;
432 audiofmt.audio.i_blockalign = audiofmt.audio.i_channels * (audiofmt.audio.i_bitspersample / 8);
433 audiofmt.i_bitrate = audiofmt.audio.i_channels * audiofmt.audio.i_rate * audiofmt.audio.i_bitspersample;
434 p_sys->i_audio_max_buffer_size = audiofmt.i_bitrate;
436 p_sys->session = [[QTCaptureSession alloc] init];
438 success = [p_sys->session addInput:audioInput error: &o_returnedAudioError];
440 msg_Err(p_demux, "the audio capture device could not be added to capture session (%ld)", [o_returnedAudioError code]);
444 success = [p_sys->session addOutput:p_sys->audiooutput error: &o_returnedAudioError];
446 msg_Err(p_demux, "audio output could not be added to capture session (%ld)", [o_returnedAudioError code]);
450 [p_sys->session startRunning];
453 p_demux->pf_demux = Demux;
454 p_demux->pf_control = Control;
455 p_demux->info.i_update = 0;
456 p_demux->info.i_title = 0;
457 p_demux->info.i_seekpoint = 0;
459 msg_Dbg(p_demux, "New audio es %d channels %dHz",
460 audiofmt.audio.i_channels, audiofmt.audio.i_rate);
462 p_sys->p_es_audio = es_out_Add(p_demux->out, &audiofmt);
464 [audioInput release];
467 msg_Dbg(p_demux, "QTSound: We have an audio device ready!");
471 [audioInput release];
479 /*****************************************************************************
480 * Close: destroy interface
481 *****************************************************************************/
482 static void Close(vlc_object_t *p_this)
484 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
485 demux_t *p_demux = (demux_t*)p_this;
486 demux_sys_t *p_sys = p_demux->p_sys;
488 [p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
489 [p_sys->audiooutput performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
490 [p_sys->session performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
497 /*****************************************************************************
499 *****************************************************************************/
500 static int Demux(demux_t *p_demux)
502 demux_sys_t *p_sys = p_demux->p_sys;
504 NSAutoreleasePool *pool;
506 p_blocka = block_Alloc(p_sys->i_audio_max_buffer_size);
509 msg_Err(p_demux, "cannot get audio block");
513 @synchronized (p_sys->audiooutput) {
514 if ([p_sys->audiooutput checkCurrentAudioBuffer]) {
515 p_blocka->i_buffer = p_blocka->i_size = [p_sys->audiooutput getCurrentTotalDataSize];
516 p_blocka->p_buffer = p_blocka->p_start = [p_sys->audiooutput getCurrentAudioBufferData];
517 p_blocka->i_nb_samples = [p_sys->audiooutput getNumberOfSamples];
518 p_blocka->i_pts = [p_sys->audiooutput getCurrentPts];
522 if(!p_blocka->i_pts) {
523 // Nothing to transfer yet, just forget
524 block_Release(p_blocka);
530 es_out_Control(p_demux->out, ES_OUT_SET_PCR, p_blocka->i_pts);
531 es_out_Send(p_demux->out, p_sys->p_es_audio, p_blocka);
534 @synchronized (p_sys->audiooutput) {
535 [p_sys->audiooutput freeAudioMem];
541 /*****************************************************************************
543 *****************************************************************************/
544 static int Control(demux_t *p_demux, int i_query, va_list args)
550 /* Special for access_demux */
551 case DEMUX_CAN_PAUSE:
553 case DEMUX_SET_PAUSE_STATE:
554 case DEMUX_CAN_CONTROL_PACE:
555 pb = (bool*)va_arg(args, bool *);
559 case DEMUX_GET_PTS_DELAY:
560 pi64 = (int64_t*)va_arg(args, int64_t *);
561 *pi64 = INT64_C(1000) * var_InheritInteger(p_demux, "live-caching");