1 /*****************************************************************************
2 * qtsound.m: qtkit (Mac OS X) based audio capture module
3 *****************************************************************************
4 * Copyright © 2011 VLC authors and VideoLAN
6 * Authors: Pierre d'Herbemont <pdherbemont@videolan.org>
7 * Gustaf Neumann <neumann@wu.ac.at>
8 * Michael S. Feurstein <michael.feurstein@wu.ac.at>
10 *****************************************************************************
11 * This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * This library is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with this library; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
25 *****************************************************************************/
27 /*****************************************************************************
29 *****************************************************************************/
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
39 #include <vlc_demux.h>
40 #include <vlc_dialog.h>
42 #import <QTKit/QTKit.h>
44 /*****************************************************************************
46 *****************************************************************************/
47 static int Open( vlc_object_t *p_this );
48 static void Close( vlc_object_t *p_this );
49 static int Demux( demux_t *p_demux );
50 static int Control( demux_t *, int, va_list );
52 /*****************************************************************************
54 *****************************************************************************/
57 set_shortname( N_("QTSound") )
58 set_description( N_("QuickTime Sound Capture") )
59 set_category( CAT_INPUT )
60 set_subcategory( SUBCAT_INPUT_ACCESS )
61 add_shortcut( "qtsound" )
62 set_capability( "access_demux", 0 )
63 set_callbacks( Open, Close )
67 /*****************************************************************************
69 *****************************************************************************/
70 @interface VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
73 AudioBuffer *currentAudioBuffer;
74 UInt32 numberOfSamples;
79 - (id)initWithDemux:(demux_t *)p_demux;
80 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
81 - (BOOL)checkCurrentAudioBuffer;
82 - (mtime_t)getCurrentPts;
83 - (void *)getCurrentAudioBufferData;
84 - (UInt32)getCurrentTotalDataSize;
85 - (UInt32)getNumberOfSamples;
89 @implementation VLCDecompressedAudioOutput : QTCaptureDecompressedAudioOutput
90 - (id)initWithDemux:(demux_t *)p_demux
92 if( self = [super init] )
95 currentAudioBuffer = nil;
96 date_Init(&date, 44100, 1);
107 free(currentAudioBuffer);
108 currentAudioBuffer = nil;
113 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection
115 AudioBufferList *tempAudioBufferList;
116 block_t *rawAudioData;
117 UInt32 totalDataSize = 0;
122 numberOfSamples = [sampleBuffer numberOfSamples];
123 date_Increment(&date,numberOfSamples);
124 currentPts = date_Get(&date);
126 tempAudioBufferList = [sampleBuffer audioBufferListWithOptions:0];
127 if (tempAudioBufferList->mNumberBuffers == 2)
130 * Compute totalDataSize as sum of all data blocks in the
133 for ( count = 0; count < tempAudioBufferList->mNumberBuffers; count++ )
135 totalDataSize += tempAudioBufferList->mBuffers[count].mDataByteSize;
138 * Allocate storage for the interleaved audio data
140 rawAudioData = block_Alloc(totalDataSize * sizeof(float));
141 if (NULL == rawAudioData)
143 msg_Err( p_qtsound, "Raw audiodata could not be allocated" );
149 msg_Err( p_qtsound, "Too many or only one channel found." );
154 * Interleave raw data (provided in two separate channels as
155 * F32L) with 2 samples per frame
160 const float *b1Ptr, *b2Ptr;
164 uPtr = (float *)rawAudioData,
165 b1Ptr = (const float *) tempAudioBufferList->mBuffers[0].mData,
166 b2Ptr = (const float *) tempAudioBufferList->mBuffers[1].mData;
167 i < numberOfSamples; i++)
177 if (currentAudioBuffer == nil)
179 currentAudioBuffer = (AudioBuffer *)malloc(sizeof(AudioBuffer));
180 if (NULL == currentAudioBuffer)
182 msg_Err( p_qtsound, "AudioBuffer could not be allocated." );
186 currentAudioBuffer->mNumberChannels = 2;
187 currentAudioBuffer->mDataByteSize = totalDataSize;
188 currentAudioBuffer->mData = rawAudioData;
194 - (BOOL)checkCurrentAudioBuffer
196 return (currentAudioBuffer) ? 1 : 0;
199 - (mtime_t)getCurrentPts
201 /* FIXME: can this getter be minimized? */
204 if( !currentAudioBuffer || currentPts == previousPts )
211 pts = previousPts = currentPts;
214 return (currentAudioBuffer->mData) ? currentPts : 0;
217 - (void *)getCurrentAudioBufferData
219 return currentAudioBuffer->mData;
222 - (UInt32)getCurrentTotalDataSize
224 return currentAudioBuffer->mDataByteSize;
227 - (UInt32)getNumberOfSamples
229 return numberOfSamples;
234 /*****************************************************************************
236 *****************************************************************************/
239 QTCaptureSession * session;
240 QTCaptureDevice * audiodevice;
241 VLCDecompressedAudioOutput * audiooutput;
242 es_out_id_t *p_es_audio;
243 int i_audio_max_buffer_size;
246 /*****************************************************************************
247 * Open: initialize interface
248 *****************************************************************************/
249 static int Open( vlc_object_t *p_this )
251 demux_t *p_demux = (demux_t*)p_this;
253 es_format_t audiofmt;
254 char *psz_uid = NULL;
257 NSString *qtk_curraudiodevice_uid;
258 NSAutoreleasePool *pool;
259 NSArray *myAudioDevices, *audioformat_array;
260 QTFormatDescription *audio_format;
261 QTCaptureDeviceInput *audioInput;
262 NSError *o_returnedAudioError;
264 if( p_demux->psz_location && *p_demux->psz_location )
266 psz_uid = p_demux->psz_location;
268 msg_Dbg( p_demux, "qtsound uid = %s", psz_uid );
269 qtk_curraudiodevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
271 pool = [[NSAutoreleasePool alloc] init];
273 p_demux->p_sys = p_sys = calloc( 1, sizeof( demux_sys_t ) );
277 msg_Dbg( p_demux, "qtsound : uid = %s", [qtk_curraudiodevice_uid UTF8String]);
278 myAudioDevices = [[[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeSound] arrayByAddingObjectsFromArray:[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeMuxed]] retain];
279 if([myAudioDevices count] == 0)
281 dialog_FatalWait( p_demux, _("No Audio Input device found"),
282 _("Your Mac does not seem to be equipped with a suitable audio input device."
283 "Please check your connectors and drivers.") );
284 msg_Err( p_demux, "Can't find any Audio device" );
289 for(iaudio = 0; iaudio < [myAudioDevices count]; iaudio++){
290 QTCaptureDevice *qtk_audioDevice;
291 qtk_audioDevice = [myAudioDevices objectAtIndex:iaudio];
292 msg_Dbg( p_demux, "qtsound audio %d/%d localizedDisplayName: %s uniqueID: %s", iaudio, [myAudioDevices count], [[qtk_audioDevice localizedDisplayName] UTF8String], [[qtk_audioDevice uniqueID] UTF8String]);
293 if([[[qtk_audioDevice localizedDisplayName]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:qtk_curraudiodevice_uid]){
294 msg_Dbg( p_demux, "Device found" );
300 if(iaudio < [myAudioDevices count]){
301 p_sys->audiodevice = [myAudioDevices objectAtIndex:iaudio];
305 /* cannot find designated audio device, fall back to open default audio device */
306 msg_Dbg(p_demux, "Cannot find designated uid audio device as %s. Fall back to open default audio device.", [qtk_curraudiodevice_uid UTF8String]);
307 p_sys->audiodevice = [QTCaptureDevice defaultInputDeviceWithMediaType: QTMediaTypeSound];
309 if( !p_sys->audiodevice )
311 dialog_FatalWait( p_demux, _("No audio input device found"),
312 _("Your Mac does not seem to be equipped with a suitable audio input device."
313 "Please check your connectors and drivers.") );
314 msg_Err( p_demux, "Can't find any Audio device" );
319 if( ![p_sys->audiodevice open: &o_returnedAudioError] )
321 msg_Err( p_demux, "Unable to open the audio capture device (%d)", [o_returnedAudioError code] );
325 if( [p_sys->audiodevice isInUseByAnotherApplication] == YES )
327 msg_Err( p_demux, "default audio capture device is exclusively in use by another application" );
330 audioInput = [[QTCaptureDeviceInput alloc] initWithDevice: p_sys->audiodevice];
333 msg_Err( p_demux, "can't create a valid audio capture input facility" );
336 msg_Dbg( p_demux, "created valid audio capture input facility" );
339 p_sys->audiooutput = [[VLCDecompressedAudioOutput alloc] initWithDemux:p_demux];
340 msg_Dbg ( p_demux, "initialized audio output" );
342 /* Get the formats */
344 FIXME: the format description gathered here does not seem to be the same
345 in comparison to the format description collected from the actual sampleBuffer.
346 This information needs to be updated some other place. For the time being this shall suffice.
348 The following verbose output is an example of what is read from the input device during the below block
349 [0x3042138] qtsound demux debug: Audio localized format summary: Linear PCM, 24 bit little-endian signed integer, 2 channels, 44100 Hz
350 [0x3042138] qtsound demux debug: Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000004; Bytes per Packet: 8; Frames per Packet: 1; Bytes per Frame: 8; Channels per Frame: 2; Bits per Channel: 24
351 [0x3042138] qtsound demux debug: Flag float 0 bigEndian 0 signedInt 1 packed 0 alignedHigh 0 non interleaved 0 non mixable 0
352 canonical 0 nativeFloatPacked 0 nativeEndian 0
354 However when reading this information from the sampleBuffer during the delegate call from
355 - (void)outputAudioSampleBuffer:(QTSampleBuffer *)sampleBuffer fromConnection:(QTCaptureConnection *)connection;
356 the following data shows up
357 2011-09-23 22:06:03.077 VLC[23070:f103] Audio localized format summary: Linear PCM, 32 bit little-endian floating point, 2 channels, 44100 Hz
358 2011-09-23 22:06:03.078 VLC[23070:f103] Sample Rate: 44100; Format ID: lpcm; Format Flags: 00000029; Bytes per Packet: 4; Frames per Packet: 1; Bytes per Frame: 4; Channels per Frame: 2; Bits per Channel: 32
359 2011-09-23 22:06:03.078 VLC[23070:f103] Flag float 1 bigEndian 0 signedInt 0 packed 1 alignedHigh 0 non interleaved 1 non mixable 0
360 canonical 1 nativeFloatPacked 1 nativeEndian 0
364 little-endian signed integer vs. little-endian floating point
365 format flag 00000004 vs. 00000029
366 bytes per packet 8 vs. 4
368 non interleaved 0 vs. 1 -> this makes a major difference when filling our own buffer
370 nativeFloatPacked 0 vs. 1
372 One would assume we'd need to feed the (es_format_t)audiofmt with the data collected here.
373 This is not the case. Audio will be transmitted in artefacts, due to wrong information.
375 At the moment this data is set manually, however one should consider trying to set this data dynamically
377 audioformat_array = [p_sys->audiodevice formatDescriptions];
379 for( int k = 0; k < [audioformat_array count]; k++ )
381 audio_format = (QTFormatDescription *)[audioformat_array objectAtIndex: k];
383 msg_Dbg( p_demux, "Audio localized format summary: %s", [[audio_format localizedFormatSummary] UTF8String]);
384 msg_Dbg( p_demux, "Audio format description attributes: %s",[[[audio_format formatDescriptionAttributes] description] UTF8String]);
386 AudioStreamBasicDescription asbd = {0};
387 NSValue *asbdValue = [audio_format attributeForKey:QTFormatDescriptionAudioStreamBasicDescriptionAttribute];
388 [asbdValue getValue:&asbd];
390 char formatIDString[5];
391 UInt32 formatID = CFSwapInt32HostToBig (asbd.mFormatID);
392 bcopy (&formatID, formatIDString, 4);
393 formatIDString[4] = '\0';
395 /* kept for development purposes */
397 msg_Dbg( p_demux, "Sample Rate: %.0lf; Format ID: %s; Format Flags: %.8x; Bytes per Packet: %d; Frames per Packet: %d; Bytes per Frame: %d; Channels per Frame: %d; Bits per Channel: %d",
401 asbd.mBytesPerPacket,
402 asbd.mFramesPerPacket,
404 asbd.mChannelsPerFrame,
405 asbd.mBitsPerChannel);
407 msg_Dbg( p_demux, "Flag float %d bigEndian %d signedInt %d packed %d alignedHigh %d non interleaved %d non mixable %d\ncanonical %d nativeFloatPacked %d nativeEndian %d",
408 (asbd.mFormatFlags & kAudioFormatFlagIsFloat) != 0,
409 (asbd.mFormatFlags & kAudioFormatFlagIsBigEndian) != 0,
410 (asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0,
411 (asbd.mFormatFlags & kAudioFormatFlagIsPacked) != 0,
412 (asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh) != 0,
413 (asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0,
414 (asbd.mFormatFlags & kAudioFormatFlagIsNonMixable) != 0,
416 (asbd.mFormatFlags & kAudioFormatFlagsCanonical) != 0,
417 (asbd.mFormatFlags & kAudioFormatFlagsNativeFloatPacked) != 0,
418 (asbd.mFormatFlags & kAudioFormatFlagsNativeEndian) != 0
423 if( [audioformat_array count] )
424 audio_format = [audioformat_array objectAtIndex: 0];
427 /* Now we can init */
428 audiocodec = VLC_CODEC_FL32;
429 es_format_Init( &audiofmt, AUDIO_ES, audiocodec);
431 audiofmt.audio.i_format = audiocodec;
432 audiofmt.audio.i_rate = 44100;
434 * i_physical_channels Describes the channels configuration of the
435 * samples (ie. number of channels which are available in the
436 * buffer, and positions).
438 audiofmt.audio.i_physical_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
440 * i_original_channels Describes from which original channels,
441 * before downmixing, the buffer is derived.
443 audiofmt.audio.i_original_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
445 * i_bytes_per_frame Optional - for A/52, SPDIF and DTS types:
446 * Bytes used by one compressed frame, depends on bitrate.
448 audiofmt.audio.i_bytes_per_frame = 4;
450 * Number of sampleframes contained in one compressed frame.
452 audiofmt.audio.i_frame_length = 1;
454 * Please note that it may be completely arbitrary - buffers are not
455 * obliged to contain a integral number of so-called "frames". It's
456 * just here for the division:
457 * buffer_size = i_nb_samples * i_bytes_per_frame / i_frame_length
459 audiofmt.audio.i_bitspersample = 32;
460 audiofmt.audio.i_channels = 2;
461 audiofmt.audio.i_blockalign = audiofmt.audio.i_channels * audiofmt.audio.i_bitspersample / 16;
462 audiofmt.i_bitrate = audiofmt.audio.i_channels * audiofmt.audio.i_rate * audiofmt.audio.i_bitspersample;
463 p_sys->i_audio_max_buffer_size = 4096;
465 p_sys->session = [[QTCaptureSession alloc] init];
467 success = [p_sys->session addInput:audioInput error: &o_returnedAudioError];
470 msg_Err( p_demux, "the audio capture device could not be added to capture session (%d)", [o_returnedAudioError code] );
474 success = [p_sys->session addOutput:p_sys->audiooutput error: &o_returnedAudioError];
477 msg_Err( p_demux, "audio output could not be added to capture session (%d)", [o_returnedAudioError code] );
481 [p_sys->session startRunning];
484 p_demux->pf_demux = Demux;
485 p_demux->pf_control = Control;
486 p_demux->info.i_update = 0;
487 p_demux->info.i_title = 0;
488 p_demux->info.i_seekpoint = 0;
490 msg_Dbg( p_demux, "New audio es %d channels %dHz",
491 audiofmt.audio.i_channels, audiofmt.audio.i_rate );
493 p_sys->p_es_audio = es_out_Add( p_demux->out, &audiofmt );
495 [audioInput release];
498 msg_Dbg( p_demux, "QTSound: We have an audio device ready!" );
502 [audioInput release];
510 /*****************************************************************************
511 * Close: destroy interface
512 *****************************************************************************/
513 static void Close( vlc_object_t *p_this )
515 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
516 demux_t *p_demux = (demux_t*)p_this;
517 demux_sys_t *p_sys = p_demux->p_sys;
519 [p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
520 [p_sys->audiooutput performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
521 [p_sys->session performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
528 /*****************************************************************************
530 *****************************************************************************/
531 static int Demux( demux_t *p_demux )
533 demux_sys_t *p_sys = p_demux->p_sys;
535 NSAutoreleasePool *pool;
537 p_blocka = block_New( p_demux, p_sys->i_audio_max_buffer_size );
541 msg_Err( p_demux, "cannot get audio block" );
545 pool = [[NSAutoreleasePool alloc] init];
547 @synchronized (p_sys->audiooutput)
549 if ( [p_sys->audiooutput checkCurrentAudioBuffer] )
551 p_blocka->i_pts = [p_sys->audiooutput getCurrentPts];
552 p_blocka->p_buffer = [p_sys->audiooutput getCurrentAudioBufferData];
553 p_blocka->i_nb_samples = [p_sys->audiooutput getNumberOfSamples];
554 p_blocka->i_buffer = [p_sys->audiooutput getCurrentTotalDataSize];
558 if( !p_blocka->i_pts )
560 // Nothing to transfer yet, just forget
561 block_Release( p_blocka );
571 es_out_Control( p_demux->out, ES_OUT_SET_PCR, p_blocka->i_pts );
572 es_out_Send( p_demux->out, p_sys->p_es_audio, p_blocka );
578 /*****************************************************************************
580 *****************************************************************************/
581 static int Control( demux_t *p_demux, int i_query, va_list args )
588 /* Special for access_demux */
589 case DEMUX_CAN_PAUSE:
591 case DEMUX_SET_PAUSE_STATE:
592 case DEMUX_CAN_CONTROL_PACE:
593 pb = (bool*)va_arg( args, bool * );
597 case DEMUX_GET_PTS_DELAY:
598 pi64 = (int64_t*)va_arg( args, int64_t * );
599 *pi64 = INT64_C(1000) * var_InheritInteger( p_demux, "live-caching" );