Fixed bug #896 John Popplewell 2009-12-08 23:05:50 PST Originally reported by AKFoerster on the mailing list. Error decoding UTF8 Russian text to UTF-16LE on Windows, but specifically on platforms without iconv support (the default on Windows). Valid UTF8 characters are flagged as being overlong and then substituted by the UNKNOWN_UNICODE character. After studying the testiconv.c example program, reading the RFCs and putting some printf statements in SDL_iconv.c the problem is in a test for 'Maximum overlong sequences', specifically 4.2.1, which is carried out by the following code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xCE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { Here is the 2-byte encoding of a character in range 00000080 - 000007FF 110xxxxx 10xxxxxx The line in question is supposed to be checking for an overlong sequence which would be less than 11000001 10111111 which should be represented as a single byte. BUT, the mask value (0xCE) is wrong, it isn't checking the top-most bit: 11000001 value 11001110 mask (incorrect) ^ and should be (0xDE): 11000001 value 11011110 mask (correct) making the above code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xDE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { I can supply a test program and/or a patch if required, best regards, John Popplewell

/* Simple test of the SDL semaphore code */

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>

#include "SDL.h"
#include "SDL_thread.h"

#define NUM_THREADS 10

static SDL_sem *sem;
int alive = 1;

int SDLCALL ThreadFunc(void *data)
{
	int threadnum = (int)(uintptr_t)data;
	while ( alive ) {
		SDL_SemWait(sem);
		fprintf(stderr, "Thread number %d has got the semaphore (value = %d)!\n", threadnum, SDL_SemValue(sem));
		SDL_Delay(200);
		SDL_SemPost(sem);
		fprintf(stderr, "Thread number %d has released the semaphore (value = %d)!\n", threadnum, SDL_SemValue(sem));
		SDL_Delay(1); /* For the scheduler */
	}
	printf("Thread number %d exiting.\n", threadnum);
	return 0;
}

static void killed(int sig)
{
	alive = 0;
}

int main(int argc, char **argv)
{
	SDL_Thread *threads[NUM_THREADS];
	uintptr_t i;
	int init_sem;

	if(argc < 2) {
		fprintf(stderr,"Usage: %s init_value\n", argv[0]);
		return(1);
	}

	/* Load the SDL library */
	if ( SDL_Init(0) < 0 ) {
		fprintf(stderr, "Couldn't initialize SDL: %s\n",SDL_GetError());
		return(1);
	}
	signal(SIGTERM, killed);
	signal(SIGINT, killed);
	
	init_sem = atoi(argv[1]);
	sem = SDL_CreateSemaphore(init_sem);
	
	printf("Running %d threads, semaphore value = %d\n", NUM_THREADS, init_sem);
	/* Create all the threads */
	for( i = 0; i < NUM_THREADS; ++i ) {
		threads[i] = SDL_CreateThread(ThreadFunc, (void*)i);
	}

	/* Wait 10 seconds */
	SDL_Delay(10 * 1000);

	/* Wait for all threads to finish */
	printf("Waiting for threads to finish\n");
	alive = 0;
	for( i = 0; i < NUM_THREADS; ++i ) {
		SDL_WaitThread(threads[i], NULL);
	}
	printf("Finished waiting for threads\n");

	SDL_DestroySemaphore(sem);
	SDL_Quit();
	return(0);
}