1 /*****************************************************************************
2 * qtsound.m: qtkit (Mac OS X) based audio capture module
3 *****************************************************************************
4 * Copyright © 2011 VLC authors and VideoLAN
6 * Authors: Pierre d'Herbemont <pdherbemont@videolan.org>
7 * Gustaf Neumann <neumann@wu.ac.at>
8 * Michael S. Feurstein <michael.feurstein@wu.ac.at>
10 *****************************************************************************
11 * This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * This library is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with this library; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
25 *****************************************************************************/
27 /*****************************************************************************
29 *****************************************************************************/
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
39 #include <vlc_demux.h>
40 #include <vlc_dialog.h>
42 //#define QTKIT_VERSION_MIN_REQUIRED 70603
43 #define QTKIT_VERSION_MAX_ALLOWED 70700
45 #import <QTKit/QTKit.h>
47 /*****************************************************************************
49 *****************************************************************************/
50 static int Open( vlc_object_t *p_this );
51 static void Close( vlc_object_t *p_this );
52 static int Demux( demux_t *p_demux );
53 static int Control( demux_t *, int, va_list );
55 /*****************************************************************************
57 *****************************************************************************/
60 set_shortname( N_("QTSound") )
61 set_description( N_("QuickTime Sound Capture") )
62 set_category( CAT_INPUT )
63 set_subcategory( SUBCAT_INPUT_ACCESS )
64 add_shortcut( "qtsound" )
65 set_capability( "access_demux", 0 )
66 set_callbacks( Open, Close )
70 /*****************************************************************************
72 *****************************************************************************/
73 @interface VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
76 AudioBuffer *currentAudioBuffer;
77 block_t *rawAudioData;
78 UInt32 numberOfSamples;
83 - (id)initWithDemux:(demux_t *)p_demux;
84 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
85 - (BOOL)checkCurrentAudioBuffer;
87 - (mtime_t)getCurrentPts;
88 - (void *)getCurrentAudioBufferData;
89 - (UInt32)getCurrentTotalDataSize;
90 - (UInt32)getNumberOfSamples;
94 @implementation VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
95 - (id)initWithDemux:(demux_t *)p_demux
97 if( self = [super init] )
100 currentAudioBuffer = nil;
101 date_Init(&date, 44100, 1);
113 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection
115 AudioBufferList *tempAudioBufferList;
116 UInt32 totalDataSize = 0;
121 numberOfSamples = [sampleBuffer numberOfSamples];
122 date_Increment(&date,numberOfSamples);
123 currentPts = date_Get(&date);
125 tempAudioBufferList = [sampleBuffer audioBufferListWithOptions:0];
126 if (tempAudioBufferList->mNumberBuffers == 2)
129 * Compute totalDataSize as sum of all data blocks in the
132 for ( count = 0; count < tempAudioBufferList->mNumberBuffers; count++ )
134 totalDataSize += tempAudioBufferList->mBuffers[count].mDataByteSize;
137 * Allocate storage for the interleaved audio data
139 rawAudioData = block_Alloc(totalDataSize * sizeof(float));
140 if (NULL == rawAudioData)
142 msg_Err( p_qtsound, "Raw audiodata could not be allocated" );
148 msg_Err( p_qtsound, "Too many or only one channel found." );
153 * Interleave raw data (provided in two separate channels as
154 * F32L) with 2 samples per frame
159 const float *b1Ptr, *b2Ptr;
163 uPtr = (float *)rawAudioData,
164 b1Ptr = (const float *) tempAudioBufferList->mBuffers[0].mData,
165 b2Ptr = (const float *) tempAudioBufferList->mBuffers[1].mData;
166 i < numberOfSamples; i++)
176 if (currentAudioBuffer == nil)
178 currentAudioBuffer = (AudioBuffer *)malloc(sizeof(AudioBuffer));
179 if (NULL == currentAudioBuffer)
181 msg_Err( p_qtsound, "AudioBuffer could not be allocated." );
185 currentAudioBuffer->mNumberChannels = 2;
186 currentAudioBuffer->mDataByteSize = totalDataSize;
187 currentAudioBuffer->mData = rawAudioData;
192 - (BOOL)checkCurrentAudioBuffer
194 return (currentAudioBuffer) ? 1 : 0;
207 - (mtime_t)getCurrentPts
209 /* FIXME: can this getter be minimized? */
212 if( !currentAudioBuffer || currentPts == previousPts )
219 pts = previousPts = currentPts;
222 return (currentAudioBuffer->mData) ? currentPts : 0;
225 - (void *)getCurrentAudioBufferData
227 return currentAudioBuffer->mData;
230 - (UInt32)getCurrentTotalDataSize
232 return currentAudioBuffer->mDataByteSize;
235 - (UInt32)getNumberOfSamples
237 return numberOfSamples;
242 /*****************************************************************************
244 *****************************************************************************/
247 QTCaptureSession * session;
248 QTCaptureDevice * audiodevice;
249 VLCDecompressedAudioOutput * audiooutput;
250 es_out_id_t *p_es_audio;
251 int i_audio_max_buffer_size;
254 /*****************************************************************************
255 * Open: initialize interface
256 *****************************************************************************/
257 static int Open( vlc_object_t *p_this )
259 demux_t *p_demux = (demux_t*)p_this;
261 es_format_t audiofmt;
262 char *psz_uid = NULL;
265 NSString *qtk_curraudiodevice_uid;
266 NSAutoreleasePool *pool;
267 NSArray *myAudioDevices, *audioformat_array;
268 QTFormatDescription *audio_format;
269 QTCaptureDeviceInput *audioInput;
270 NSError *o_returnedAudioError;
272 if( p_demux->psz_location && *p_demux->psz_location )
274 psz_uid = p_demux->psz_location;
276 msg_Dbg( p_demux, "qtsound uid = %s", psz_uid );
277 qtk_curraudiodevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
279 pool = [[NSAutoreleasePool alloc] init];
281 p_demux->p_sys = p_sys = calloc( 1, sizeof( demux_sys_t ) );
285 msg_Dbg( p_demux, "qtsound : uid = %s", [qtk_curraudiodevice_uid UTF8String]);
286 myAudioDevices = [[[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeSound] arrayByAddingObjectsFromArray:[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeMuxed]] retain];
287 if([myAudioDevices count] == 0)
289 dialog_FatalWait( p_demux, _("No Audio Input device found"),
290 _("Your Mac does not seem to be equipped with a suitable audio input device."
291 "Please check your connectors and drivers.") );
292 msg_Err( p_demux, "Can't find any Audio device" );
297 for(iaudio = 0; iaudio < [myAudioDevices count]; iaudio++){
298 QTCaptureDevice *qtk_audioDevice;
299 qtk_audioDevice = [myAudioDevices objectAtIndex:iaudio];
300 msg_Dbg( p_demux, "qtsound audio %u/%lu localizedDisplayName: %s uniqueID: %s", iaudio, [myAudioDevices count], [[qtk_audioDevice localizedDisplayName] UTF8String], [[qtk_audioDevice uniqueID] UTF8String]);
301 if([[[qtk_audioDevice uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:qtk_curraudiodevice_uid]){
302 msg_Dbg( p_demux, "Device found" );
308 if(iaudio < [myAudioDevices count]){
309 p_sys->audiodevice = [myAudioDevices objectAtIndex:iaudio];
313 /* cannot find designated audio device, fall back to open default audio device */
314 msg_Dbg(p_demux, "Cannot find designated uid audio device as %s. Fall back to open default audio device.", [qtk_curraudiodevice_uid UTF8String]);
315 p_sys->audiodevice = [QTCaptureDevice defaultInputDeviceWithMediaType: QTMediaTypeSound];
317 if( !p_sys->audiodevice )
319 dialog_FatalWait( p_demux, _("No audio input device found"),
320 _("Your Mac does not seem to be equipped with a suitable audio input device."
321 "Please check your connectors and drivers.") );
322 msg_Err( p_demux, "Can't find any Audio device" );
327 if( ![p_sys->audiodevice open: &o_returnedAudioError] )
329 msg_Err( p_demux, "Unable to open the audio capture device (%ld)", [o_returnedAudioError code] );
333 if( [p_sys->audiodevice isInUseByAnotherApplication] == YES )
335 msg_Err( p_demux, "default audio capture device is exclusively in use by another application" );
338 audioInput = [[QTCaptureDeviceInput alloc] initWithDevice: p_sys->audiodevice];
341 msg_Err( p_demux, "can't create a valid audio capture input facility" );
344 msg_Dbg( p_demux, "created valid audio capture input facility" );
347 p_sys->audiooutput = [[VLCDecompressedAudioOutput alloc] initWithDemux:p_demux];
348 msg_Dbg ( p_demux, "initialized audio output" );
350 /* Get the formats */
352 FIXME: the format description gathered here does not seem to be the same
353 in comparison to the format description collected from the actual sampleBuffer.
354 This information needs to be updated some other place. For the time being this shall suffice.
356 The following verbose output is an example of what is read from the input device during the below block
357 [0x3042138] qtsound demux debug: Audio localized format summary: Linear PCM, 24 bit little-endian signed integer, 2 channels, 44100 Hz
358 [0x3042138] qtsound demux debug: Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000004; Bytes per Packet: 8; Frames per Packet: 1; Bytes per Frame: 8; Channels per Frame: 2; Bits per Channel: 24
359 [0x3042138] qtsound demux debug: Flag float 0 bigEndian 0 signedInt 1 packed 0 alignedHigh 0 non interleaved 0 non mixable 0
360 canonical 0 nativeFloatPacked 0 nativeEndian 0
362 However when reading this information from the sampleBuffer during the delegate call from
363 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
364 the following data shows up
365 2011-09-23 22:06:03.077 VLC[23070:f103] Audio localized format summary: Linear PCM, 32 bit little-endian floating point, 2 channels, 44100 Hz
366 2011-09-23 22:06:03.078 VLC[23070:f103] Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000029; Bytes per Packet: 4; Frames per Packet: 1; Bytes per Frame: 4; Channels per Frame: 2; Bits per Channel: 32
367 2011-09-23 22:06:03.078 VLC[23070:f103] Flag float 1 bigEndian 0 signedInt 0 packed 1 alignedHigh 0 non interleaved 1 non mixable 0
368 canonical 1 nativeFloatPacked 1 nativeEndian 0
372 little-endian signed integer vs. little-endian floating point
373 format flag 00000004 vs. 00000029
374 bytes per packet 8 vs. 4
376 non interleaved 0 vs. 1 -> this makes a major difference when filling our own buffer
378 nativeFloatPacked 0 vs. 1
380 One would assume we'd need to feed the (es_format_t)audiofmt with the data collected here.
381 This is not the case. Audio will be transmitted in artefacts, due to wrong information.
383 At the moment this data is set manually, however one should consider trying to set this data dynamically
385 audioformat_array = [p_sys->audiodevice formatDescriptions];
387 for( int k = 0; k < [audioformat_array count]; k++ )
389 audio_format = (QTFormatDescription *)[audioformat_array objectAtIndex: k];
391 msg_Dbg( p_demux, "Audio localized format summary: %s", [[audio_format localizedFormatSummary] UTF8String]);
392 msg_Dbg( p_demux, "Audio format description attributes: %s",[[[audio_format formatDescriptionAttributes] description] UTF8String]);
394 AudioStreamBasicDescription asbd = {0};
395 NSValue *asbdValue = [audio_format attributeForKey:QTFormatDescriptionAudioStreamBasicDescriptionAttribute];
396 [asbdValue getValue:&asbd];
398 char formatIDString[5];
399 UInt32 formatID = CFSwapInt32HostToBig (asbd.mFormatID);
400 bcopy (&formatID, formatIDString, 4);
401 formatIDString[4] = '\0';
403 /* kept for development purposes */
405 msg_Dbg( p_demux, "Sample Rate: %.0lf; Format ID: %s; Format Flags: %.8x; Bytes per Packet: %d; Frames per Packet: %d; Bytes per Frame: %d; Channels per Frame: %d; Bits per Channel: %d",
409 asbd.mBytesPerPacket,
410 asbd.mFramesPerPacket,
412 asbd.mChannelsPerFrame,
413 asbd.mBitsPerChannel);
415 msg_Dbg( p_demux, "Flag float %d bigEndian %d signedInt %d packed %d alignedHigh %d non interleaved %d non mixable %d\ncanonical %d nativeFloatPacked %d nativeEndian %d",
416 (asbd.mFormatFlags & kAudioFormatFlagIsFloat) != 0,
417 (asbd.mFormatFlags & kAudioFormatFlagIsBigEndian) != 0,
418 (asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0,
419 (asbd.mFormatFlags & kAudioFormatFlagIsPacked) != 0,
420 (asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh) != 0,
421 (asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0,
422 (asbd.mFormatFlags & kAudioFormatFlagIsNonMixable) != 0,
424 (asbd.mFormatFlags & kAudioFormatFlagsCanonical) != 0,
425 (asbd.mFormatFlags & kAudioFormatFlagsNativeFloatPacked) != 0,
426 (asbd.mFormatFlags & kAudioFormatFlagsNativeEndian) != 0
431 if( [audioformat_array count] )
432 audio_format = [audioformat_array objectAtIndex: 0];
435 /* Now we can init */
436 audiocodec = VLC_CODEC_FL32;
437 es_format_Init( &audiofmt, AUDIO_ES, audiocodec);
439 audiofmt.audio.i_format = audiocodec;
440 audiofmt.audio.i_rate = 44100;
442 * i_physical_channels Describes the channels configuration of the
443 * samples (ie. number of channels which are available in the
444 * buffer, and positions).
446 audiofmt.audio.i_physical_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
448 * i_original_channels Describes from which original channels,
449 * before downmixing, the buffer is derived.
451 audiofmt.audio.i_original_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
453 * i_bytes_per_frame Optional - for A/52, SPDIF and DTS types:
454 * Bytes used by one compressed frame, depends on bitrate.
456 audiofmt.audio.i_bytes_per_frame = 4;
458 * Number of sampleframes contained in one compressed frame.
460 audiofmt.audio.i_frame_length = 1;
462 * Please note that it may be completely arbitrary - buffers are not
463 * obliged to contain a integral number of so-called "frames". It's
464 * just here for the division:
465 * buffer_size = i_nb_samples * i_bytes_per_frame / i_frame_length
467 audiofmt.audio.i_bitspersample = 32;
468 audiofmt.audio.i_channels = 2;
469 audiofmt.audio.i_blockalign = audiofmt.audio.i_channels * audiofmt.audio.i_bitspersample / 16;
470 audiofmt.i_bitrate = audiofmt.audio.i_channels * audiofmt.audio.i_rate * audiofmt.audio.i_bitspersample;
471 p_sys->i_audio_max_buffer_size = 4096;
473 p_sys->session = [[QTCaptureSession alloc] init];
475 success = [p_sys->session addInput:audioInput error: &o_returnedAudioError];
478 msg_Err( p_demux, "the audio capture device could not be added to capture session (%ld)", [o_returnedAudioError code] );
482 success = [p_sys->session addOutput:p_sys->audiooutput error: &o_returnedAudioError];
485 msg_Err( p_demux, "audio output could not be added to capture session (%ld)", [o_returnedAudioError code] );
489 [p_sys->session startRunning];
492 p_demux->pf_demux = Demux;
493 p_demux->pf_control = Control;
494 p_demux->info.i_update = 0;
495 p_demux->info.i_title = 0;
496 p_demux->info.i_seekpoint = 0;
498 msg_Dbg( p_demux, "New audio es %d channels %dHz",
499 audiofmt.audio.i_channels, audiofmt.audio.i_rate );
501 p_sys->p_es_audio = es_out_Add( p_demux->out, &audiofmt );
503 [audioInput release];
506 msg_Dbg( p_demux, "QTSound: We have an audio device ready!" );
510 [audioInput release];
518 /*****************************************************************************
519 * Close: destroy interface
520 *****************************************************************************/
521 static void Close( vlc_object_t *p_this )
523 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
524 demux_t *p_demux = (demux_t*)p_this;
525 demux_sys_t *p_sys = p_demux->p_sys;
527 [p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
528 [p_sys->audiooutput performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
529 [p_sys->session performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
536 /*****************************************************************************
538 *****************************************************************************/
539 static int Demux( demux_t *p_demux )
541 demux_sys_t *p_sys = p_demux->p_sys;
543 NSAutoreleasePool *pool;
545 p_blocka = block_New( p_demux, p_sys->i_audio_max_buffer_size );
549 msg_Err( p_demux, "cannot get audio block" );
553 @synchronized (p_sys->audiooutput)
555 if ( [p_sys->audiooutput checkCurrentAudioBuffer] )
557 p_blocka->i_pts = [p_sys->audiooutput getCurrentPts];
558 p_blocka->p_buffer = [p_sys->audiooutput getCurrentAudioBufferData];
559 p_blocka->i_nb_samples = [p_sys->audiooutput getNumberOfSamples];
560 p_blocka->i_buffer = [p_sys->audiooutput getCurrentTotalDataSize];
564 if( !p_blocka->i_pts )
566 // Nothing to transfer yet, just forget
567 block_Release( p_blocka );
574 es_out_Control( p_demux->out, ES_OUT_SET_PCR, p_blocka->i_pts );
575 es_out_Send( p_demux->out, p_sys->p_es_audio, p_blocka );
578 @synchronized (p_sys->audiooutput)
583 * Wait before freeing memory, so we don't get no crackling sound
584 * crackling sound artefacts start at 100 ms and below
587 [p_sys->audiooutput freeAudioMem];
593 /*****************************************************************************
595 *****************************************************************************/
596 static int Control( demux_t *p_demux, int i_query, va_list args )
603 /* Special for access_demux */
604 case DEMUX_CAN_PAUSE:
606 case DEMUX_SET_PAUSE_STATE:
607 case DEMUX_CAN_CONTROL_PACE:
608 pb = (bool*)va_arg( args, bool * );
612 case DEMUX_GET_PTS_DELAY:
613 pi64 = (int64_t*)va_arg( args, int64_t * );
614 *pi64 = INT64_C(1000) * var_InheritInteger( p_demux, "live-caching" );