X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Fmisc%2Funicode.c;h=3e90cbb5daa8b799571393dc2ace8e7d5629d3f6;hb=4a3e1087fb3804e3fca54c6a86b8c4b65cfeef25;hp=c58c54cc9a365a2d5ff4b07214d4bfb6a172b667;hpb=f075541729c8c384215c0479d01671cefab4af69;p=vlc diff --git a/src/misc/unicode.c b/src/misc/unicode.c index c58c54cc9a..3e90cbb5da 100644 --- a/src/misc/unicode.c +++ b/src/misc/unicode.c @@ -1,11 +1,14 @@ /***************************************************************************** - * unicode.c: UTF8 <-> locale functions + * unicode.c: Unicode <-> locale functions ***************************************************************************** * Copyright (C) 2005-2006 the VideoLAN team * $Id$ * * Authors: Rémi Denis-Courmont * + * UTF16toUTF8() adapted from Perl 5 (also GPL'd) + * Copyright (C) 1998-2002, Larry Wall + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -30,12 +33,20 @@ #include #include +#include +#include #include #include -#include +#ifdef HAVE_DIRENT_H +# include +#endif +#ifdef UNDER_CE +# include +#endif #ifdef HAVE_SYS_STAT_H # include #endif + #ifndef HAVE_LSTAT # define lstat( a, b ) stat(a, b) #endif @@ -45,16 +56,17 @@ # define ASSUME_UTF8 1 #endif -#if !(defined (WIN32) && defined (UNDER_CE) && defined (ASSUME_UTF8)) -# define USE_ICONV 1 -#endif - -#if defined (USE_ICONV) && !defined (HAVE_ICONV) -# error No UTF8 charset conversion implemented on this platform! +#ifndef ASSUME_UTF8 +# if defined (HAVE_ICONV) +/* libiconv is more powerful than Win32 API (it has translit) */ +# define USE_ICONV 1 +# elif defined (WIN32) || defined (UNDER_CE) +# define USE_MB2MB 1 +# else +# error No UTF8 charset conversion implemented on this platform! +# endif #endif - - #ifdef USE_ICONV static struct { vlc_iconv_t hd; @@ -73,19 +85,25 @@ void LocaleInit( vlc_object_t *p_this ) else { /* not UTF-8 */ - char *psz_conv = psz_charset; + char psz_buf[strlen( psz_charset ) + sizeof( "//translit" )]; + const char *psz_conv; /* * Still allow non-ASCII characters when the locale is not set. * Western Europeans are being favored for historical reasons. */ - psz_conv = strcmp( psz_charset, "ASCII" ) - ? psz_charset : "ISO-8859-1"; + if( strcmp( psz_charset, "ASCII" ) ) + { + sprintf( psz_buf, "%s//translit", psz_charset ); + psz_conv = psz_buf; + } + else + psz_conv = "ISO-8859-1//translit"; vlc_mutex_init( p_this, &from_locale.lock ); vlc_mutex_init( p_this, &to_locale.lock ); - from_locale.hd = vlc_iconv_open( "UTF-8", psz_charset ); - to_locale.hd = vlc_iconv_open( psz_charset, "UTF-8" ); + from_locale.hd = vlc_iconv_open( "UTF-8", psz_conv ); + to_locale.hd = vlc_iconv_open( psz_conv, "UTF-8" ); } free( psz_charset ); @@ -114,57 +132,52 @@ void LocaleDeinit( void ) #endif } -#if defined (WIN32) || defined (UNDER_CE) +#ifdef USE_MB2MB static char *MB2MB( const char *string, UINT fromCP, UINT toCP ) { char *out; - int ilen = strlen( string ), olen = (4 / sizeof (wchar_t)) * ilen + 1; - wchar_t wide[olen]; + wchar_t *wide; + int len; - ilen = MultiByteToWideChar( fromCP, 0, string, ilen + 1, wide, olen ); - if( ilen == 0 ) + len = MultiByteToWideChar( fromCP, 0, string, -1, NULL, 0 ); + if( len == 0 ) return NULL; - olen = 4 * ilen + 1; - out = malloc( olen ); + wchar_t wide[len]; - olen = WideCharToMultiByte( toCP, 0, wide, ilen, out, olen, NULL, NULL ); - if( olen == 0 ) - { - free( out ); + MultiByteToWideChar( fromCP, 0, string, -1, wide, len ); + len = WideCharToMultiByte( toCP, 0, wide, -1, NULL, 0, NULL, NULL ); + if( len == 0 ) return NULL; - } - return realloc( out, olen ); + out = malloc( len ); + + WideCharToMultiByte( toCP, 0, wide, -1, out, len, NULL, NULL ); + return out; } #endif -/***************************************************************************** +/** * FromLocale: converts a locale string to UTF-8 - *****************************************************************************/ + * + * @param locale nul-terminated string to be converted + * + * @return a nul-terminated UTF-8 string, or NULL in case of error. + * To avoid memory leak, you have to pass the result to LocaleFree() + * when it is no longer needed. + */ char *FromLocale( const char *locale ) { if( locale == NULL ) return NULL; -#if !(defined WIN32 || defined (UNDER_CE)) +#ifndef USE_MB2MB # ifdef USE_ICONV if( from_locale.hd != (vlc_iconv_t)(-1) ) { - char *iptr = (char *)locale, *output, *optr; - size_t inb, outb; - - /* - * We are not allowed to modify the locale pointer, even if we cast it - * to non-const. - */ - inb = strlen( locale ); - /* FIXME: I'm not sure about the value for the multiplication - * (for western people, multiplication by 3 (Latin9) is needed). - * While UTF-8 could reach 6 bytes, no existing code point exceeds - * 4 bytes. */ - outb = inb * 4 + 1; - - optr = output = malloc( outb ); + const char *iptr = locale; + size_t inb = strlen( locale ); + size_t outb = inb * 6 + 1; + char output[outb], *optr = output; vlc_mutex_lock( &from_locale.lock ); vlc_iconv( from_locale.hd, NULL, NULL, NULL, NULL ); @@ -185,11 +198,11 @@ char *FromLocale( const char *locale ) assert (*iptr == '\0'); assert (*optr == '\0'); assert (strlen( output ) == (size_t)(optr - output)); - return realloc( output, optr - output + 1 ); + return strdup( output ); } # endif /* USE_ICONV */ return (char *)locale; -#else /* WIN32 */ +#else /* MB2MB */ return MB2MB( locale, CP_ACP, CP_UTF8 ); #endif } @@ -208,31 +221,32 @@ char *FromLocaleDup( const char *locale ) } -/***************************************************************************** - * ToLocale: converts an UTF-8 string to locale - *****************************************************************************/ +/** + * ToLocale: converts a UTF-8 string to local system encoding. + * + * @param utf8 nul-terminated string to be converted + * + * @return a nul-terminated string, or NULL in case of error. + * To avoid memory leak, you have to pass the result to LocaleFree() + * when it is no longer needed. + */ char *ToLocale( const char *utf8 ) { if( utf8 == NULL ) return NULL; -#if !(defined (WIN32) || defined (UNDER_CE)) +#ifndef USE_MB2MB # ifdef USE_ICONV if( to_locale.hd != (vlc_iconv_t)(-1) ) { - char *iptr = (char *)utf8, *output, *optr; - size_t inb, outb; - - /* - * We are not allowed to modify the locale pointer, even if we cast it - * to non-const. - */ - inb = strlen( utf8 ); + const char *iptr = utf8; + size_t inb = strlen( utf8 ); /* FIXME: I'm not sure about the value for the multiplication * (for western people, multiplication is not needed) */ - outb = inb * 2 + 1; + size_t outb = inb * 2 + 1; + + char output[outb], *optr = output; - optr = output = malloc( outb ); vlc_mutex_lock( &to_locale.lock ); vlc_iconv( to_locale.hd, NULL, NULL, NULL, NULL ); @@ -252,15 +266,28 @@ char *ToLocale( const char *utf8 ) assert (*iptr == '\0'); assert (*optr == '\0'); assert (strlen( output ) == (size_t)(optr - output)); - return realloc( output, optr - output + 1 ); + return strdup( output ); } # endif /* USE_ICONV */ return (char *)utf8; -#else /* WIN32 */ +#else /* MB2MB */ return MB2MB( utf8, CP_UTF8, CP_ACP ); #endif } +char *ToLocaleDup( const char *utf8 ) +{ +#if defined (ASSUME_UTF8) + return strdup( utf8 ); +#else +# ifdef USE_ICONV + if (to_locale.hd == (vlc_iconv_t)(-1)) + return strdup( utf8 ); +# endif + return ToLocale( utf8 ); +#endif +} + void LocaleFree( const char *str ) { #ifdef USE_ICONV @@ -274,12 +301,34 @@ void LocaleFree( const char *str ) #endif } -/***************************************************************************** +/** * utf8_fopen: Calls fopen() after conversion of file name to OS locale - *****************************************************************************/ + */ FILE *utf8_fopen( const char *filename, const char *mode ) { -#if !(defined (WIN32) || defined (UNDER_CE)) +#if defined (WIN32) || defined (UNDER_CE) + if( GetVersion() < 0x80000000 ) + { + /* for Windows NT and above */ + wchar_t wpath[MAX_PATH + 1]; + size_t len = strlen( mode ) + 1; + wchar_t wmode[len]; + + if( !MultiByteToWideChar( CP_UTF8, 0, filename, -1, wpath, MAX_PATH ) + || !MultiByteToWideChar( CP_ACP, 0, mode, len, wmode, len ) ) + { + errno = ENOENT; + return NULL; + } + wpath[MAX_PATH] = L'\0'; + + /* + * fopen() cannot open files with non-“ANSI” characters on Windows. + * We use _wfopen() instead. Same thing for mkdir() and stat(). + */ + return _wfopen( wpath, wmode ); + } +#endif const char *local_name = ToLocale( filename ); if( local_name != NULL ) @@ -290,30 +339,18 @@ FILE *utf8_fopen( const char *filename, const char *mode ) } else errno = ENOENT; - return NULL; -#else - wchar_t wpath[MAX_PATH]; - wchar_t wmode[4]; - if( !MultiByteToWideChar( CP_UTF8, 0, filename, -1, wpath, MAX_PATH - 1) - || !MultiByteToWideChar( CP_ACP, 0, mode, -1, wmode, 3 ) ) - { - errno = ENOENT; - return NULL; - } - - return _wfopen( wpath, wmode ); -#endif + return NULL; } -/***************************************************************************** +/** * utf8_mkdir: Calls mkdir() after conversion of file name to OS locale - *****************************************************************************/ + */ int utf8_mkdir( const char *dirname ) { #if defined (UNDER_CE) || defined (WIN32) - wchar_t wname[MAX_PATH]; - char mod[MAX_PATH]; + wchar_t wname[MAX_PATH + 1]; + char mod[MAX_PATH + 1]; int i; /* Convert '/' into '\' */ @@ -336,12 +373,14 @@ int utf8_mkdir( const char *dirname ) errno = ENOENT; return -1; } + wname[MAX_PATH] = L'\0'; if( CreateDirectoryW( wname, NULL ) == 0 ) { if( GetLastError( ) == ERROR_ALREADY_EXISTS ) errno = EEXIST; - errno = ENOENT; + else + errno = ENOENT; return -1; } return 0; @@ -364,11 +403,12 @@ int utf8_mkdir( const char *dirname ) void *utf8_opendir( const char *dirname ) { + /* TODO: support for WinNT non-ACP filenames */ const char *local_name = ToLocale( dirname ); if( local_name != NULL ) { - DIR *dir = opendir( local_name ); + DIR *dir = vlc_opendir_wrapper( local_name ); LocaleFree( local_name ); return dir; } @@ -381,17 +421,100 @@ const char *utf8_readdir( void *dir ) { struct dirent *ent; - ent = readdir( (DIR *)dir ); + ent = vlc_readdir_wrapper( (DIR *)dir ); if( ent == NULL ) return NULL; return FromLocale( ent->d_name ); } +static int dummy_select( const char *str ) +{ + (void)str; + return 1; +} + +int utf8_scandir( const char *dirname, char ***namelist, + int (*select)( const char * ), + int (*compar)( const char **, const char ** ) ) +{ + DIR *dir = utf8_opendir( dirname ); + + if( select == NULL ) + select = dummy_select; + + if( dir == NULL ) + return -1; + else + { + char **tab = NULL; + const char *entry; + unsigned num = 0; + + while( ( entry = utf8_readdir( dir ) ) != NULL ) + { + char **newtab; + char *utf_entry = strdup( entry ); + LocaleFree( entry ); + if( utf_entry == NULL ) + goto error; + + if( !select( utf_entry ) ) + { + free( utf_entry ); + continue; + } + + newtab = realloc( tab, sizeof( char * ) * (num + 1) ); + if( newtab == NULL ) + { + free( utf_entry ); + goto error; + } + tab = newtab; + tab[num++] = utf_entry; + } + vlc_closedir_wrapper( dir ); + + if( compar != NULL ) + qsort( tab, num, sizeof( tab[0] ), + (int (*)( const void *, const void *))compar ); + + *namelist = tab; + return num; + + error:{ + unsigned i; + + for( i = 0; i < num; i++ ) + free( tab[i] ); + if( tab != NULL ) + free( tab ); + return -1;} + } +} + static int utf8_statEx( const char *filename, void *buf, vlc_bool_t deref ) { +#if defined (WIN32) || defined (UNDER_CE) + /* retrieve Windows OS version */ + if( GetVersion() < 0x80000000 ) + { + /* for Windows NT and above */ + wchar_t wpath[MAX_PATH + 1]; + + if( !MultiByteToWideChar( CP_UTF8, 0, filename, -1, wpath, MAX_PATH ) ) + { + errno = ENOENT; + return -1; + } + wpath[MAX_PATH] = L'\0'; + + return _wstati64( wpath, (struct _stati64 *)buf ); + } +#endif #ifdef HAVE_SYS_STAT_H const char *local_name = ToLocale( filename ); @@ -418,26 +541,58 @@ int utf8_lstat( const char *filename, void *buf) return utf8_statEx( filename, buf, VLC_FALSE ); } -/***************************************************************************** - * EnsureUTF8: replaces invalid/overlong UTF-8 sequences with question marks - ***************************************************************************** - * Not Todo : convert Latin1 to UTF-8 on the flu - * It is not possible given UTF-8 needs more space - *****************************************************************************/ +/** + * utf8_*printf: *printf with conversion from UTF-8 to local encoding + */ +static int utf8_vasprintf( char **str, const char *fmt, va_list ap ) +{ + char *utf8; + int res = vasprintf( &utf8, fmt, ap ); + if( res == -1 ) + return -1; + + *str = ToLocaleDup( utf8 ); + free( utf8 ); + return res; +} + +int utf8_vfprintf( FILE *stream, const char *fmt, va_list ap ) +{ + char *str; + int res = utf8_vasprintf( &str, fmt, ap ); + if( res == -1 ) + return -1; + + fputs( str, stream ); + free( str ); + return res; +} + +int utf8_fprintf( FILE *stream, const char *fmt, ... ) +{ + va_list ap; + int res; + + va_start( ap, fmt ); + res = utf8_vfprintf( stream, fmt, ap ); + va_end( ap ); + return res; +} + + +static char *CheckUTF8( char *str, char rep ) #define isutf8cont( c ) (((c) >= 0x80) && ((c) <= 0xBF)) -char *EnsureUTF8( char *str ) { unsigned char *ptr, c; + assert (str != NULL); + ptr = (unsigned char *)str; while( (c = *ptr) != '\0' ) { /* US-ASCII, 1 byte */ - if( ( ( c >= 0x20 ) && ( c <= 0x7F ) ) - || ( c == 0x09 ) || ( c == 0x0A ) || ( c == 0x0D ) ) - { + if( c <= 0x7F ) ptr++; /* OK */ - } else /* 2 bytes */ if( ( c >= 0xC2 ) && ( c <= 0xDF ) ) @@ -446,7 +601,7 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 2; /* OK */ else - *ptr++ = '?'; /* invalid */ + goto error; } else /* 3 bytes */ @@ -459,10 +614,10 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 3; /* OK */ else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else if( ( ( c >= 0xE1 ) && ( c <= 0xEC ) ) || ( c == 0xEC ) @@ -475,10 +630,10 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 3; /* OK */ else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else if( c == 0xED ) @@ -490,10 +645,10 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 3; /* OK */ else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else /* 4 bytes */ @@ -509,13 +664,13 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 4; /* OK */ else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else if( ( c >= 0xF1 ) && ( c <= 0xF3 ) ) @@ -529,14 +684,13 @@ char *EnsureUTF8( char *str ) c = ptr[3]; if( isutf8cont( c ) ) ptr += 4; /* OK */ - else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else if( c == 0xF4 ) @@ -551,25 +705,70 @@ char *EnsureUTF8( char *str ) if( isutf8cont( c ) ) ptr += 4; /* OK */ else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; } else - *ptr++ = '?'; + goto error; + + continue; + +error: + if( rep == 0 ) + return NULL; + *ptr++ = '?'; + str = NULL; } return str; } -/********************************************************************** - * UTF32toUTF8: converts an array from UTF-32 to UTF-8 - *********************************************************************/ -char *UTF32toUTF8( const wchar_t *src, size_t len, size_t *newlen ) +/** + * EnsureUTF8: replaces invalid/overlong UTF-8 sequences with question marks + * Note that it is not possible to convert from Latin-1 to UTF-8 on the fly, + * so we don't try that, even though it would be less disruptive. + * + * @return str if it was valid UTF-8, NULL if not. + */ +char *EnsureUTF8( char *str ) +{ + return CheckUTF8( str, '?' ); +} + + +/** + * IsUTF8: checks whether a string is a valid UTF-8 byte sequence. + * + * @param str nul-terminated string to be checked + * + * @return str if it was valid UTF-8, NULL if not. + */ +const char *IsUTF8( const char *str ) +{ + return CheckUTF8( (char *)str, 0 ); +} + + +/** + * UTF32toUTF8(): converts an array from UTF-32 (host byte order) + * to UTF-8. + * + * @param src the UTF-32 table to be converted + * @param len the number of code points to be converted from src + * (ie. the number of uint32_t in the table pointed to by src) + * @param newlen an optional pointer. If not NULL, *newlen will + * contain the total number of bytes written. + * + * @return the result of the conversion (must be free'd()) + * or NULL on error (in that case, *newlen is undefined). + */ +static char * +UTF32toUTF8( const uint32_t *src, size_t len, size_t *newlen ) { char *res, *out; @@ -604,6 +803,7 @@ char *UTF32toUTF8( const wchar_t *src, size_t len, size_t *newlen ) continue; } else + if( uv < 0x110000 ) { *out++ = (( uv >> 18) | 0xf0); *out++ = (((uv >> 12) & 0x3f) | 0x80); @@ -611,6 +811,11 @@ char *UTF32toUTF8( const wchar_t *src, size_t len, size_t *newlen ) *out++ = (( uv & 0x3f) | 0x80); continue; } + else + { + free( res ); + return NULL; + } } len = out - res; res = realloc( res, len ); @@ -619,19 +824,114 @@ char *UTF32toUTF8( const wchar_t *src, size_t len, size_t *newlen ) return res; } -/********************************************************************** - * FromUTF32: converts an UTF-32 string to UTF-8 - ********************************************************************** - * The result must be free()'d. NULL on error. - *********************************************************************/ -char *FromUTF32( const wchar_t *src ) +/** + * FromUTF32(): converts an UTF-32 string to UTF-8. + * + * @param src UTF-32 bytes sequence, aligned on a 32-bits boundary. + * + * @return the result of the conversion (must be free()'d), + * or NULL in case of error. + */ +char *FromUTF32( const uint32_t *src ) { + const uint32_t *in; size_t len; - const wchar_t *in; /* determine the size of the string */ - for( len = 1, in = src; GetWBE( in ); len++ ) + for( len = 1, in = src; *in; len++ ) in++; return UTF32toUTF8( src, len, NULL ); } + +/** + * UTF16toUTF8: converts UTF-16 (host byte order) to UTF-8 + * + * @param src UTF-16 bytes sequence, aligned on a 16-bits boundary + * @param len number of uint16_t to convert + */ +static char * +UTF16toUTF8( const uint16_t *in, size_t len, size_t *newlen ) +{ + char *res, *out; + + /* allocate memory */ + out = res = (char *)malloc( 3 * len ); + if( res == NULL ) + return NULL; + + while( len > 0 ) + { + uint32_t uv = *in; + + in++; + len--; + + if( uv < 0x80 ) + { + *out++ = uv; + continue; + } + if( uv < 0x800 ) + { + *out++ = (( uv >> 6) | 0xc0); + *out++ = (( uv & 0x3f) | 0x80); + continue; + } + if( (uv >= 0xd800) && (uv < 0xdbff) ) + { /* surrogates */ + uint16_t low = GetWBE( in ); + in++; + len--; + + if( (low < 0xdc00) || (low >= 0xdfff) ) + { + *out++ = '?'; /* Malformed surrogate */ + continue; + } + else + uv = ((uv - 0xd800) << 10) + (low - 0xdc00) + 0x10000; + } + if( uv < 0x10000 ) + { + *out++ = (( uv >> 12) | 0xe0); + *out++ = (((uv >> 6) & 0x3f) | 0x80); + *out++ = (( uv & 0x3f) | 0x80); + continue; + } + else + { + *out++ = (( uv >> 18) | 0xf0); + *out++ = (((uv >> 12) & 0x3f) | 0x80); + *out++ = (((uv >> 6) & 0x3f) | 0x80); + *out++ = (( uv & 0x3f) | 0x80); + continue; + } + } + len = out - res; + res = realloc( res, len ); + if( newlen != NULL ) + *newlen = len; + return res; +} + + +/** + * FromUTF16(): converts an UTF-16 string to UTF-8. + * + * @param src UTF-16 bytes sequence, aligned on a 16-bits boundary. + * + * @return the result of the conversion (must be free()'d), + * or NULL in case of error. + */ +char *FromUTF16( const uint16_t *src ) +{ + const uint16_t *in; + size_t len; + + /* determine the size of the string */ + for( len = 1, in = src; *in; len++ ) + in++; + + return UTF16toUTF8( src, len, NULL ); +}