Mercurial > sdl-ios-xcode
view test/testfile.c @ 3539:f2846bf19360
Fixed bug #896
John Popplewell 2009-12-08 23:05:50 PST
Originally reported by AKFoerster on the mailing list.
Error decoding UTF8 Russian text to UTF-16LE on Windows, but specifically on
platforms without iconv support (the default on Windows).
Valid UTF8 characters are flagged as being overlong and then substituted by the
UNKNOWN_UNICODE character.
After studying the testiconv.c example program, reading the RFCs and putting
some printf statements in SDL_iconv.c the problem is in a test for 'Maximum
overlong sequences', specifically 4.2.1, which is carried out by the following
code:
} else if ( p[0] >= 0xC0 ) {
if ( (p[0] & 0xE0) != 0xC0 ) {
/* Skip illegal sequences
return SDL_ICONV_EILSEQ;
*/
ch = UNKNOWN_UNICODE;
} else {
if ( (p[0] & 0xCE) == 0xC0 ) { <<<<<<<< here
overlong = SDL_TRUE;
}
ch = (Uint32)(p[0] & 0x1F);
left = 1;
}
} else {
Here is the 2-byte encoding of a character in range 00000080 - 000007FF
110xxxxx 10xxxxxx
The line in question is supposed to be checking for an overlong sequence which
would be less than
11000001 10111111
which should be represented as a single byte.
BUT, the mask value (0xCE) is wrong, it isn't checking the top-most bit:
11000001 value
11001110 mask (incorrect)
^
and should be (0xDE):
11000001 value
11011110 mask (correct)
making the above code:
} else if ( p[0] >= 0xC0 ) {
if ( (p[0] & 0xE0) != 0xC0 ) {
/* Skip illegal sequences
return SDL_ICONV_EILSEQ;
*/
ch = UNKNOWN_UNICODE;
} else {
if ( (p[0] & 0xDE) == 0xC0 ) { <<<<<<<< here
overlong = SDL_TRUE;
}
ch = (Uint32)(p[0] & 0x1F);
left = 1;
}
} else {
I can supply a test program and/or a patch if required,
best regards,
John Popplewell
author | Sam Lantinga <slouken@libsdl.org> |
---|---|
date | Fri, 11 Dec 2009 08:03:43 +0000 |
parents | f55c87ae336b |
children | 27ab20a36eba |
line wrap: on
line source
/* sanity tests on SDL_rwops.c (usefull for alternative implementations of stdio rwops) */ #include <stdlib.h> #ifndef _MSC_VER #include <unistd.h> #endif #include "SDL.h" #include "SDL_endian.h" #include <stdio.h> /* WARNING ! those 2 files will be destroyed by this test program */ #ifdef __IPHONEOS__ #define FBASENAME1 "../Documents/sdldata1" /* this file will be created during tests */ #define FBASENAME2 "../Documents/sdldata2" /* this file should not exist before starting test */ #else #define FBASENAME1 "sdldata1" /* this file will be created during tests */ #define FBASENAME2 "sdldata2" /* this file should not exist before starting test */ #endif #ifndef NULL #define NULL ((void *)0) #endif static void cleanup(void) { unlink(FBASENAME1); unlink(FBASENAME2); } static void rwops_error_quit(unsigned line, SDL_RWops * rwops) { printf("testfile.c(%d): failed\n", line); if (rwops) { rwops->close(rwops); /* This calls SDL_FreeRW(rwops); */ } cleanup(); exit(1); /* quit with rwops error (test failed) */ } #define RWOP_ERR_QUIT(x) rwops_error_quit( __LINE__, (x) ) int main(int argc, char *argv[]) { SDL_RWops *rwops = NULL; char test_buf[30]; cleanup(); /* test 1 : basic argument test: all those calls to SDL_RWFromFile should fail */ rwops = SDL_RWFromFile(NULL, NULL); if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile(NULL, "ab+"); if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile(NULL, "sldfkjsldkfj"); if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile("something", ""); if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile("something", NULL); if (rwops) RWOP_ERR_QUIT(rwops); printf("test1 OK\n"); /* test 2 : check that inexistant file is not successfully opened/created when required */ /* modes : r, r+ implie that file MUST exist modes : a, a+, w, w+ checks that it succeeds (file may not exists) */ rwops = SDL_RWFromFile(FBASENAME2, "rb"); /* this file doesn't exist that call must fail */ if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile(FBASENAME2, "rb+"); /* this file doesn't exist that call must fail */ if (rwops) RWOP_ERR_QUIT(rwops); rwops = SDL_RWFromFile(FBASENAME2, "wb"); if (!rwops) RWOP_ERR_QUIT(rwops); rwops->close(rwops); unlink(FBASENAME2); rwops = SDL_RWFromFile(FBASENAME2, "wb+"); if (!rwops) RWOP_ERR_QUIT(rwops); rwops->close(rwops); unlink(FBASENAME2); rwops = SDL_RWFromFile(FBASENAME2, "ab"); if (!rwops) RWOP_ERR_QUIT(rwops); rwops->close(rwops); unlink(FBASENAME2); rwops = SDL_RWFromFile(FBASENAME2, "ab+"); if (!rwops) RWOP_ERR_QUIT(rwops); rwops->close(rwops); unlink(FBASENAME2); printf("test2 OK\n"); /* test 3 : creation, writing , reading, seeking, test : w mode, r mode, w+ mode */ rwops = SDL_RWFromFile(FBASENAME1, "wb"); /* write only */ if (!rwops) RWOP_ERR_QUIT(rwops); if (1 != rwops->write(rwops, "1234567890", 10, 1)) RWOP_ERR_QUIT(rwops); if (10 != rwops->write(rwops, "1234567890", 1, 10)) RWOP_ERR_QUIT(rwops); if (7 != rwops->write(rwops, "1234567", 1, 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); /* we are in write only mode */ rwops->close(rwops); rwops = SDL_RWFromFile(FBASENAME1, "rb"); /* read mode, file must exists */ if (!rwops) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (20 != rwops->seek(rwops, -7, RW_SEEK_END)) RWOP_ERR_QUIT(rwops); if (7 != rwops->read(rwops, test_buf, 1, 7)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "1234567", 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 10, 100)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, -27, RW_SEEK_CUR)) RWOP_ERR_QUIT(rwops); if (2 != rwops->read(rwops, test_buf, 10, 3)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "12345678901234567890", 20)) RWOP_ERR_QUIT(rwops); if (0 != rwops->write(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); /* readonly mode */ rwops->close(rwops); /* test 3: same with w+ mode */ rwops = SDL_RWFromFile(FBASENAME1, "wb+"); /* write + read + truncation */ if (!rwops) RWOP_ERR_QUIT(rwops); if (1 != rwops->write(rwops, "1234567890", 10, 1)) RWOP_ERR_QUIT(rwops); if (10 != rwops->write(rwops, "1234567890", 1, 10)) RWOP_ERR_QUIT(rwops); if (7 != rwops->write(rwops, "1234567", 1, 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (1 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); /* we are in read/write mode */ if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (20 != rwops->seek(rwops, -7, RW_SEEK_END)) RWOP_ERR_QUIT(rwops); if (7 != rwops->read(rwops, test_buf, 1, 7)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "1234567", 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 10, 100)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, -27, RW_SEEK_CUR)) RWOP_ERR_QUIT(rwops); if (2 != rwops->read(rwops, test_buf, 10, 3)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "12345678901234567890", 20)) RWOP_ERR_QUIT(rwops); rwops->close(rwops); printf("test3 OK\n"); /* test 4: same in r+ mode */ rwops = SDL_RWFromFile(FBASENAME1, "rb+"); /* write + read + file must exists, no truncation */ if (!rwops) RWOP_ERR_QUIT(rwops); if (1 != rwops->write(rwops, "1234567890", 10, 1)) RWOP_ERR_QUIT(rwops); if (10 != rwops->write(rwops, "1234567890", 1, 10)) RWOP_ERR_QUIT(rwops); if (7 != rwops->write(rwops, "1234567", 1, 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (1 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); /* we are in read/write mode */ if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (20 != rwops->seek(rwops, -7, RW_SEEK_END)) RWOP_ERR_QUIT(rwops); if (7 != rwops->read(rwops, test_buf, 1, 7)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "1234567", 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 10, 100)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, -27, RW_SEEK_CUR)) RWOP_ERR_QUIT(rwops); if (2 != rwops->read(rwops, test_buf, 10, 3)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "12345678901234567890", 20)) RWOP_ERR_QUIT(rwops); rwops->close(rwops); printf("test4 OK\n"); /* test5 : append mode */ rwops = SDL_RWFromFile(FBASENAME1, "ab+"); /* write + read + append */ if (!rwops) RWOP_ERR_QUIT(rwops); if (1 != rwops->write(rwops, "1234567890", 10, 1)) RWOP_ERR_QUIT(rwops); if (10 != rwops->write(rwops, "1234567890", 1, 10)) RWOP_ERR_QUIT(rwops); if (7 != rwops->write(rwops, "1234567", 1, 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (1 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (20 + 27 != rwops->seek(rwops, -7, RW_SEEK_END)) RWOP_ERR_QUIT(rwops); if (7 != rwops->read(rwops, test_buf, 1, 7)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "1234567", 7)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 1, 1)) RWOP_ERR_QUIT(rwops); if (0 != rwops->read(rwops, test_buf, 10, 100)) RWOP_ERR_QUIT(rwops); if (27 != rwops->seek(rwops, -27, RW_SEEK_CUR)) RWOP_ERR_QUIT(rwops); if (0 != rwops->seek(rwops, 0L, RW_SEEK_SET)) RWOP_ERR_QUIT(rwops); if (3 != rwops->read(rwops, test_buf, 10, 3)) RWOP_ERR_QUIT(rwops); if (SDL_memcmp(test_buf, "123456789012345678901234567123", 30)) RWOP_ERR_QUIT(rwops); rwops->close(rwops); printf("test5 OK\n"); cleanup(); return 0; /* all ok */ }