Fixed bug #896 John Popplewell 2009-12-08 23:05:50 PST Originally reported by AKFoerster on the mailing list. Error decoding UTF8 Russian text to UTF-16LE on Windows, but specifically on platforms without iconv support (the default on Windows). Valid UTF8 characters are flagged as being overlong and then substituted by the UNKNOWN_UNICODE character. After studying the testiconv.c example program, reading the RFCs and putting some printf statements in SDL_iconv.c the problem is in a test for 'Maximum overlong sequences', specifically 4.2.1, which is carried out by the following code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xCE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { Here is the 2-byte encoding of a character in range 00000080 - 000007FF 110xxxxx 10xxxxxx The line in question is supposed to be checking for an overlong sequence which would be less than 11000001 10111111 which should be represented as a single byte. BUT, the mask value (0xCE) is wrong, it isn't checking the top-most bit: 11000001 value 11001110 mask (incorrect) ^ and should be (0xDE): 11000001 value 11011110 mask (correct) making the above code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xDE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { I can supply a test program and/or a patch if required, best regards, John Popplewell

CFLAGS  := -W -Wall -Wextra -g `sdl-config --cflags`
LDFLAGS := `sdl-config --libs`

# If it doesn't pick up defaults
#CFLAGS  := -D_GNU_SOURCE=1 -D_REENTRANT -I/usr/local/include/SDL
#LDFLAGS := -lm -ldl -lesd -lpthread

SRC         	:= testsdl.c \
						rwops/rwops.c \
						platform/platform.c \
						surface/surface.c \
						render/render.c \
						audio/audio.c
COMMON_SRC     := SDL_at.c common/common.c
COMMON_INCLUDE := SDL_at.h

TESTS_ALL := testsdl \
	rwops/rwops \
	platform/platform \
	surface/surface \
	render/render \
	audio/audio


.PHONY: all clean test


all: $(TESTS_ALL)

test: all
	@./testsdl

testsdl: $(SRC) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(SRC) $(COMMON_SRC)

rwops/rwops: rwops/rwops.c $(COMMON_INCLUDE) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ rwops/rwops.c $(COMMON_SRC) -DTEST_STANDALONE

platform/platform: platform/platform.c $(COMMON_INCLUDE) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ platform/platform.c $(COMMON_SRC) -DTEST_STANDALONE

surface/surface: surface/surface.c $(COMMON_INCLUDE) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ surface/surface.c $(COMMON_SRC) -DTEST_STANDALONE

render/render: render/render.c $(COMMON_INCLUDE) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ render/render.c $(COMMON_SRC) -DTEST_STANDALONE

audio/audio: audio/audio.c $(COMMON_INCLUDE) $(COMMON_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ audio/audio.c $(COMMON_SRC) -DTEST_STANDALONE

clean:
	$(RM) $(TESTS_ALL)