X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Ftext%2Funicode.c;h=bed5f1f96d7cbd91693690e78297357c1bea7a62;hb=9ce1a13fb8fa77a38e0dfdfe3387829e1df3f085;hp=df0df105a31b8ef0caff60e3fca969fd19caaa5e;hpb=78277aedce99a6ca8f69844b5a93b70aa95a7ede;p=vlc diff --git a/src/text/unicode.c b/src/text/unicode.c index df0df105a3..bed5f1f96d 100644 --- a/src/text/unicode.c +++ b/src/text/unicode.c @@ -2,7 +2,7 @@ * unicode.c: Unicode <-> locale functions ***************************************************************************** * Copyright (C) 2005-2006 the VideoLAN team - * Copyright © 2005-2008 Rémi Denis-Courmont + * Copyright © 2005-2010 Rémi Denis-Courmont * * Authors: Rémi Denis-Courmont * @@ -30,84 +30,54 @@ #include #include -#include "libvlc.h" /* utf8_mkdir */ #include #include #include #include -#include #include -#ifdef HAVE_DIRENT_H -# include -#endif #ifdef UNDER_CE # include #endif -#ifdef HAVE_SYS_STAT_H -# include -#endif -#ifdef HAVE_FCNTL_H -# include -#endif -#ifdef WIN32 -# include -#else -# include -#endif - -#ifndef HAVE_LSTAT -# define lstat( a, b ) stat(a, b) -#endif - -#ifdef __APPLE__ -/* Define this if the OS always use UTF-8 internally */ -# define ASSUME_UTF8 1 -#endif +#include #if defined (ASSUME_UTF8) /* Cool */ + #elif defined (WIN32) || defined (UNDER_CE) # define USE_MB2MB 1 -#elif defined (HAVE_ICONV) -# define USE_ICONV 1 -#else -# error No UTF8 charset conversion implemented on this platform! -#endif - -#if defined (USE_ICONV) -# include -static char charset[sizeof ("CSISO11SWEDISHFORNAMES")] = ""; +# include -static void find_charset_once (void) +static char *locale_dup (const char *string, bool from) { - strlcpy (charset, nl_langinfo (CODESET), sizeof (charset)); - if (!strcasecmp (charset, "ASCII") - || !strcasecmp (charset, "ANSI_X3.4-1968")) - strcpy (charset, "UTF-8"); /* superset... */ -} + char *out; + int len; -static int find_charset (void) -{ - static pthread_once_t once = PTHREAD_ONCE_INIT; - pthread_once (&once, find_charset_once); - return !strcasecmp (charset, "UTF-8"); + len = 1 + MultiByteToWideChar (from ? CP_ACP : CP_UTF8, + 0, string, -1, NULL, 0); + wchar_t *wide = malloc (len * sizeof (wchar_t)); + if (wide == NULL) + return NULL; + + MultiByteToWideChar (from ? CP_ACP : CP_UTF8, 0, string, -1, wide, len); + len = 1 + WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1, + NULL, 0, NULL, NULL); + out = malloc (len); + if (out != NULL) + WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1, out, len, + NULL, NULL); + free (wide); + return out; } -#endif +#elif defined (HAVE_ICONV) +# define USE_ICONV 1 -static char *locale_fast (const char *string, bool from) +static char *locale_dup (const char *string, bool from) { - if( string == NULL ) - return NULL; - -#if defined (USE_ICONV) - if (find_charset ()) - return (char *)string; - - vlc_iconv_t hd = vlc_iconv_open (from ? "UTF-8" : charset, - from ? charset : "UTF-8"); + vlc_iconv_t hd = vlc_iconv_open (from ? "UTF-8" : "", + from ? "" : "UTF-8"); if (hd == (vlc_iconv_t)(-1)) return NULL; /* Uho! */ @@ -132,46 +102,12 @@ static char *locale_fast (const char *string, bool from) assert (*optr == '\0'); assert (strlen (output) == (size_t)(optr - output)); return strdup (output); -#elif defined (USE_MB2MB) - char *out; - int len; - - len = 1 + MultiByteToWideChar (from ? CP_ACP : CP_UTF8, - 0, string, -1, NULL, 0); - wchar_t wide[len]; - - MultiByteToWideChar (from ? CP_ACP : CP_UTF8, 0, string, -1, wide, len); - len = 1 + WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1, - NULL, 0, NULL, NULL); - out = malloc (len); - if (out == NULL) - return NULL; - - WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1, out, len, - NULL, NULL); - return out; -#else - (void)from; - return (char *)string; -#endif } - -static inline char *locale_dup (const char *string, bool from) -{ - assert( string ); - -#if defined (USE_ICONV) - if (find_charset ()) - return strdup (string); - return locale_fast (string, from); -#elif defined (USE_MB2MB) - return locale_fast (string, from); #else - (void)from; - return strdup (string); +# error No UTF8 charset conversion implemented on this platform! #endif -} + /** * Releases (if needed) a localized or uniformized string. @@ -179,13 +115,10 @@ static inline char *locale_dup (const char *string, bool from) */ void LocaleFree (const char *str) { -#if defined (USE_ICONV) - if (!find_charset ()) - free ((char *)str); -#elif defined (USE_MB2MB) - free ((char *)str); +#ifdef ASSUME_UTF8 + (void) str; #else - (void)str; + free ((char *)str); #endif } @@ -201,7 +134,11 @@ void LocaleFree (const char *str) */ char *FromLocale (const char *locale) { - return locale_fast (locale, true); +#ifdef ASSUME_UTF8 + return (char *)locale; +#else + return locale ? locale_dup (locale, true) : NULL; +#endif } /** @@ -215,7 +152,11 @@ char *FromLocale (const char *locale) */ char *FromLocaleDup (const char *locale) { +#ifdef ASSUME_UTF8 + return strdup (locale); +#else return locale_dup (locale, true); +#endif } @@ -230,7 +171,11 @@ char *FromLocaleDup (const char *locale) */ char *ToLocale (const char *utf8) { - return locale_fast (utf8, false); +#ifdef ASSUME_UTF8 + return (char *)utf8; +#else + return utf8 ? locale_dup (utf8, false) : NULL; +#endif } @@ -245,460 +190,71 @@ char *ToLocale (const char *utf8) */ char *ToLocaleDup (const char *utf8) { +#ifdef ASSUME_UTF8 + return strdup (utf8); +#else return locale_dup (utf8, false); -} - - -/** - * Opens a system file handle using UTF-8 paths. - * - * @param filename file path to open (with UTF-8 encoding) - * @param flags open() flags, see the C library open() documentation - * @param mode file permissions if creating a new file - * @return a file handle on success, -1 on error (see errno). - */ -int utf8_open (const char *filename, int flags, mode_t mode) -{ -#ifdef UNDER_CE - /*_open translates to wchar internally on WinCE*/ - return _open (filename, flags, mode); -#elif defined (WIN32) - /* for Windows NT and above */ - wchar_t wpath[MAX_PATH + 1]; - - if (!MultiByteToWideChar (CP_UTF8, 0, filename, -1, wpath, MAX_PATH)) - { - errno = ENOENT; - return -1; - } - wpath[MAX_PATH] = L'\0'; - - /* - * open() cannot open files with non-“ANSI” characters on Windows. - * We use _wopen() instead. Same thing for mkdir() and stat(). - */ - return _wopen (wpath, flags, mode); - #endif - const char *local_name = ToLocale (filename); - - if (local_name == NULL) - { - errno = ENOENT; - return -1; - } - - int fd = open (local_name, flags, mode); - LocaleFree (local_name); - return fd; } /** - * Opens a FILE pointer using UTF-8 filenames. - * @param filename file path, using UTF-8 encoding - * @param mode fopen file open mode - * @return NULL on error, an open FILE pointer on success. - */ -FILE *utf8_fopen (const char *filename, const char *mode) -{ - int rwflags = 0, oflags = 0; - bool append = false; - - for (const char *ptr = mode; *ptr; ptr++) - { - switch (*ptr) - { - case 'r': - rwflags = O_RDONLY; - break; - - case 'a': - rwflags = O_WRONLY; - oflags |= O_CREAT; - append = true; - break; - - case 'w': - rwflags = O_WRONLY; - oflags |= O_CREAT | O_TRUNC; - break; - - case '+': - rwflags = O_RDWR; - break; - -#ifdef O_TEXT - case 't': - oflags |= O_TEXT; - break; -#endif - } - } - - int fd = utf8_open (filename, rwflags | oflags, 0666); - if (fd == -1) - return NULL; - - if (append && (lseek (fd, 0, SEEK_END) == -1)) - { - close (fd); - return NULL; - } - - FILE *stream = fdopen (fd, mode); - if (stream == NULL) - close (fd); - - return stream; -} - -/** - * Creates a directory using UTF-8 paths. - * - * @param dirname a UTF-8 string with the name of the directory that you - * want to create. - * @param mode directory permissions - * @return 0 on success, -1 on error (see errno). + * Formats an UTF-8 string as vfprintf(), then print it, with + * appropriate conversion to local encoding. */ -int utf8_mkdir( const char *dirname, mode_t mode ) +int utf8_vfprintf( FILE *stream, const char *fmt, va_list ap ) { -#if defined (UNDER_CE) || defined (WIN32) - VLC_UNUSED( mode ); - - wchar_t wname[MAX_PATH + 1]; - char mod[MAX_PATH + 1]; - int i; - - /* Convert '/' into '\' */ - for( i = 0; *dirname; i++ ) - { - if( i == MAX_PATH ) - return -1; /* overflow */ - - if( *dirname == '/' ) - mod[i] = '\\'; - else - mod[i] = *dirname; - dirname++; - - } - mod[i] = 0; - - if( MultiByteToWideChar( CP_UTF8, 0, mod, -1, wname, MAX_PATH ) == 0 ) - { - errno = ENOENT; - return -1; - } - wname[MAX_PATH] = L'\0'; - - if( CreateDirectoryW( wname, NULL ) == 0 ) - { - if( GetLastError( ) == ERROR_ALREADY_EXISTS ) - errno = EEXIST; - else - errno = ENOENT; - return -1; - } - return 0; +#ifdef ASSUME_UTF8 + return vfprintf (stream, fmt, ap); #else - char *locname = ToLocale( dirname ); + char *str; int res; - if( locname == NULL ) - { - errno = ENOENT; - return -1; - } - res = mkdir( locname, mode ); - - LocaleFree( locname ); - return res; -#endif -} - -/** - * Opens a DIR pointer using UTF-8 paths - * - * @param dirname UTF-8 representation of the directory name - * @return a pointer to the DIR struct, or NULL in case of error. - * Release with standard closedir(). - */ -DIR *utf8_opendir( const char *dirname ) -{ -#ifdef WIN32 - wchar_t wname[MAX_PATH + 1]; - - if (MultiByteToWideChar (CP_UTF8, 0, dirname, -1, wname, MAX_PATH)) - { - wname[MAX_PATH] = L'\0'; - return (DIR *)vlc_wopendir (wname); - } -#else - const char *local_name = ToLocale( dirname ); - - if( local_name != NULL ) - { - DIR *dir = opendir( local_name ); - LocaleFree( local_name ); - return dir; - } -#endif - - errno = ENOENT; - return NULL; -} - -/** - * Reads the next file name from an open directory. - * - * @param dir The directory that is being read - * - * @return a UTF-8 string of the directory entry. - * Use free() to free this memory. - */ -char *utf8_readdir( DIR *dir ) -{ -#ifdef WIN32 - struct _wdirent *ent = vlc_wreaddir (dir); - if (ent == NULL) - return NULL; - - return FromWide (ent->d_name); -#else - struct dirent *ent; - - ent = readdir( (DIR *)dir ); - if( ent == NULL ) - return NULL; - - return vlc_fix_readdir( ent->d_name ); -#endif -} - -static int dummy_select( const char *str ) -{ - (void)str; - return 1; -} - -/** - * Does the same as utf8_scandir(), but takes an open directory pointer - * instead of a directory path. - */ -int utf8_loaddir( DIR *dir, char ***namelist, - int (*select)( const char * ), - int (*compar)( const char **, const char ** ) ) -{ - if( select == NULL ) - select = dummy_select; - - if( dir == NULL ) - return -1; - else +# if defined( WIN32 ) && !defined( UNDER_CE ) + /* Writing to the console is a lot of fun on Microsoft Windows. + * If you use the standard I/O functions, you must use the OEM code page, + * which is different from the usual ANSI code page. Or maybe not, if the + * user called "chcp". Anyway, we prefer Unicode. */ + int fd = _fileno (stream); + if (likely(fd != -1) && _isatty (fd)) { - char **tab = NULL; - char *entry; - unsigned num = 0; - - rewinddir( dir ); + res = vasprintf (&str, fmt, ap); + if (unlikely(res == -1)) + return -1; - while( ( entry = utf8_readdir( dir ) ) != NULL ) + size_t wlen = 2 * (res + 1); + wchar_t *wide = malloc (wlen); + if (likely(wide != NULL)) { - char **newtab; - - if( !select( entry ) ) + wlen = MultiByteToWideChar (CP_UTF8, 0, str, res + 1, wide, wlen); + if (wlen > 0) { - free( entry ); - continue; - } + HANDLE h = (HANDLE)(intptr_t)_get_osfhandle (fd); + DWORD out; - newtab = realloc( tab, sizeof( char * ) * (num + 1) ); - if( newtab == NULL ) - { - free( entry ); - goto error; + WriteConsoleW (h, wide, wlen - 1, &out, NULL); } - tab = newtab; - tab[num++] = entry; - } - - if( compar != NULL ) - qsort( tab, num, sizeof( tab[0] ), - (int (*)( const void *, const void *))compar ); - - *namelist = tab; - return num; - - error:{ - unsigned i; - - for( i = 0; i < num; i++ ) - free( tab[i] ); - if( tab != NULL ) - free( tab ); + else + res = -1; + free (wide); } - } - return -1; -} - -/** - * Selects file entries from a directory, as GNU C scandir(), yet using - * UTF-8 file names. - * - * @param dirname UTF-8 diretory path - * @param pointer [OUT] pointer set, on succesful completion, to the address - * of a table of UTF-8 filenames. All filenames must be freed with free(). - * The table itself must be freed with free() as well. - * - * @return How many file names were selected (possibly 0), - * or -1 in case of error. - */ -int utf8_scandir( const char *dirname, char ***namelist, - int (*select)( const char * ), - int (*compar)( const char **, const char ** ) ) -{ - DIR *dir = utf8_opendir (dirname); - int val = -1; - - if (dir != NULL) - { - val = utf8_loaddir (dir, namelist, select, compar); - closedir (dir); - } - return val; -} - -static int utf8_statEx( const char *filename, struct stat *buf, - bool deref ) -{ -#ifdef UNDER_CE - /*_stat translates to wchar internally on WinCE*/ - return _stat( filename, buf ); -#elif defined (WIN32) - /* for Windows NT and above */ - wchar_t wpath[MAX_PATH + 1]; - - if( !MultiByteToWideChar( CP_UTF8, 0, filename, -1, wpath, MAX_PATH ) ) - { - errno = ENOENT; - return -1; - } - wpath[MAX_PATH] = L'\0'; - - return _wstati64( wpath, buf ); - -#endif -#ifdef HAVE_SYS_STAT_H - const char *local_name = ToLocale( filename ); - - if( local_name != NULL ) - { - int res = deref ? stat( local_name, buf ) - : lstat( local_name, buf ); - LocaleFree( local_name ); + else + res = -1; + free (str); return res; } - errno = ENOENT; -#endif - return -1; -} - -/** - * Finds file/inode informations, as stat(). - * Consider usign fstat() instead, if possible. - * - * @param filename UTF-8 file path - */ -int utf8_stat( const char *filename, struct stat *buf) -{ - return utf8_statEx( filename, buf, true ); -} +# endif -/** - * Finds file/inode informations, as lstat(). - * Consider usign fstat() instead, if possible. - * - * @param filename UTF-8 file path - */ -int utf8_lstat( const char *filename, struct stat *buf) -{ - return utf8_statEx( filename, buf, false ); -} - -/** - * Removes a file. - * - * @param filename a UTF-8 string with the name of the file you want to delete. - * @return A 0 return value indicates success. A -1 return value indicates an - * error, and an error code is stored in errno - */ -int utf8_unlink( const char *filename ) -{ -#ifdef UNDER_CE - /*_open translates to wchar internally on WinCE*/ - return _unlink( filename ); -#elif defined (WIN32) - /* for Windows NT and above */ - wchar_t wpath[MAX_PATH + 1]; - - if( !MultiByteToWideChar( CP_UTF8, 0, filename, -1, wpath, MAX_PATH ) ) - { - errno = ENOENT; + res = vasprintf (&str, fmt, ap); + if (unlikely(res == -1)) return -1; - } - wpath[MAX_PATH] = L'\0'; - /* - * unlink() cannot open files with non-“ANSI” characters on Windows. - * We use _wunlink() instead. - */ - return _wunlink( wpath ); -#endif - const char *local_name = ToLocale( filename ); - - if( local_name == NULL ) - { - errno = ENOENT; - return -1; - } + char *ansi = ToLocaleDup (str); + free (str); - int ret = unlink( local_name ); - LocaleFree( local_name ); - return ret; -} - - - -/** - * Formats an UTF-8 string as vasprintf(), then print it to stdout, with - * appropriate conversion to local encoding. - */ -static int utf8_vasprintf( char **str, const char *fmt, va_list ap ) -{ - char *utf8; - int res = vasprintf( &utf8, fmt, ap ); - if( res == -1 ) - return -1; - - *str = ToLocaleDup( utf8 ); - free( utf8 ); - return res; -} - -/** - * Formats an UTF-8 string as vfprintf(), then print it, with - * appropriate conversion to local encoding. - */ -int utf8_vfprintf( FILE *stream, const char *fmt, va_list ap ) -{ - char *str; - int res = utf8_vasprintf( &str, fmt, ap ); - if( res == -1 ) - return -1; - - fputs( str, stream ); - free( str ); + fputs (ansi, stream); + free (ansi); return res; +#endif } /** @@ -717,70 +273,74 @@ int utf8_fprintf( FILE *stream, const char *fmt, ... ) } -static char *CheckUTF8( char *str, char rep ) +/** + * Converts the first character from a UTF-8 sequence into a code point. + * + * @param str an UTF-8 bytes sequence + * @return 0 if str points to an empty string, i.e. the first character is NUL; + * number of bytes that the first character occupies (from 1 to 4) otherwise; + * -1 if the byte sequence was not a valid UTF-8 sequence. + */ +static size_t vlc_towc (const char *str, uint32_t *restrict pwc) { uint8_t *ptr = (uint8_t *)str; assert (str != NULL); - for (;;) - { - uint8_t c = ptr[0]; - int charlen = -1; - - if (c == '\0') - break; + uint8_t c = ptr[0]; - for (int i = 0; i < 7; i++) - if ((c >> (7 - i)) == ((0xff >> (7 - i)) ^ 1)) - { - charlen = i; - break; - } - - switch (charlen) - { - case 0: // 7-bit ASCII character -> OK - ptr++; - continue; - - case -1: // 1111111x -> error - case 1: // continuation byte -> error - goto error; - } + if (unlikely(c == '\0')) + { + *pwc = 0; + return 0; + } - assert (charlen >= 2); + if (unlikely(c > 0xF4)) + return -1; - uint32_t cp = c & ~((0xff >> (7 - charlen)) << (7 - charlen)); - for (int i = 1; i < charlen; i++) - { - assert (cp < (1 << 26)); - c = ptr[i]; + int charlen = clz8 (c ^ 0xFF); + switch (charlen) + { + case 0: // 7-bit ASCII character -> OK + *pwc = c; + return 1; - if ((c == '\0') // unexpected end of string - || ((c >> 6) != 2)) // not a continuation byte - goto error; + case 1: // continuation byte -> error + return -1; + } - cp = (cp << 6) | (ptr[i] & 0x3f); - } + assert (charlen >= 2 && charlen <= 4); - if (cp < 128) // overlong (special case for ASCII) - goto error; - if (cp < (1u << (5 * charlen - 3))) // overlong - goto error; + uint32_t cp = c & ~((0xff >> (7 - charlen)) << (7 - charlen)); + for (int i = 1; i < charlen; i++) + { + assert (cp < (1 << 26)); + c = ptr[i]; - ptr += charlen; - continue; + if (unlikely((c >> 6) != 2)) // not a continuation byte + return -1; - error: - if (rep == 0) - return NULL; - *ptr++ = rep; - str = NULL; + cp = (cp << 6) | (ptr[i] & 0x3f); } - return str; + switch (charlen) + { + case 4: + if (unlikely(cp > 0x10FFFF)) // beyond Unicode + return -1; + case 3: + if (unlikely(cp >= 0xD800 && cp < 0xC000)) // UTF-16 surrogate + return -1; + case 2: + if (unlikely(cp < 128)) // ASCII overlong + return -1; + if (unlikely(cp < (1u << (5 * charlen - 3)))) // overlong + return -1; + } + *pwc = cp; + return charlen; } + /** * Replaces invalid/overlong UTF-8 sequences with question marks. * Note that it is not possible to convert from Latin-1 to UTF-8 on the fly, @@ -790,7 +350,19 @@ static char *CheckUTF8( char *str, char rep ) */ char *EnsureUTF8( char *str ) { - return CheckUTF8( str, '?' ); + char *ret = str; + size_t n; + uint32_t cp; + + while ((n = vlc_towc (str, &cp)) != 0) + if (likely(n != (size_t)-1)) + str += n; + else + { + *str++ = '?'; + ret = NULL; + } + return ret; } @@ -803,5 +375,50 @@ char *EnsureUTF8( char *str ) */ const char *IsUTF8( const char *str ) { - return CheckUTF8( (char *)str, 0 ); + size_t n; + uint32_t cp; + + while ((n = vlc_towc (str, &cp)) != 0) + if (likely(n != (size_t)-1)) + str += n; + else + return NULL; + return str; +} + +/** + * Converts a string from the given character encoding to utf-8. + * + * @return a nul-terminated utf-8 string, or null in case of error. + * The result must be freed using free(). + */ +char *FromCharset(const char *charset, const void *data, size_t data_size) +{ + vlc_iconv_t handle = vlc_iconv_open ("UTF-8", charset); + if (handle == (vlc_iconv_t)(-1)) + return NULL; + + char *out = NULL; + for(unsigned mul = 4; mul < 8; mul++ ) + { + size_t in_size = data_size; + const char *in = data; + size_t out_max = mul * data_size; + char *tmp = out = malloc (1 + out_max); + if (!out) + break; + + if (vlc_iconv (handle, &in, &in_size, &tmp, &out_max) != (size_t)(-1)) { + *tmp = '\0'; + break; + } + free(out); + out = NULL; + + if (errno != E2BIG) + break; + } + vlc_iconv_close(handle); + return out; } +