Fixed bug #896 John Popplewell 2009-12-08 23:05:50 PST Originally reported by AKFoerster on the mailing list. Error decoding UTF8 Russian text to UTF-16LE on Windows, but specifically on platforms without iconv support (the default on Windows). Valid UTF8 characters are flagged as being overlong and then substituted by the UNKNOWN_UNICODE character. After studying the testiconv.c example program, reading the RFCs and putting some printf statements in SDL_iconv.c the problem is in a test for 'Maximum overlong sequences', specifically 4.2.1, which is carried out by the following code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xCE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { Here is the 2-byte encoding of a character in range 00000080 - 000007FF 110xxxxx 10xxxxxx The line in question is supposed to be checking for an overlong sequence which would be less than 11000001 10111111 which should be represented as a single byte. BUT, the mask value (0xCE) is wrong, it isn't checking the top-most bit: 11000001 value 11001110 mask (incorrect) ^ and should be (0xDE): 11000001 value 11011110 mask (correct) making the above code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xDE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { I can supply a test program and/or a patch if required, best regards, John Popplewell

/* A simple test program framework */

#include <SDL/SDL.h>

#define VERBOSE_VIDEO   0x00000001
#define VERBOSE_MODES   0x00000002
#define VERBOSE_RENDER  0x00000004
#define VERBOSE_EVENT   0x00000008
#define VERBOSE_AUDIO   0x00000010

typedef struct
{
    /* SDL init flags */
    char **argv;
    Uint32 flags;
    Uint32 verbose;

    /* Video info */
    const char *videodriver;
    int display;
    const char *window_title;
    Uint32 window_flags;
    int window_x;
    int window_y;
    int window_w;
    int window_h;
    int depth;
    int refresh_rate;
    int num_windows;
    SDL_WindowID *windows;

    /* Renderer info */
    const char *renderdriver;
    Uint32 render_flags;
    SDL_bool skip_renderer;

    /* Audio info */
    const char *audiodriver;
    SDL_AudioSpec audiospec;
} CommonState;

extern CommonState *CommonCreateState(char **argv, Uint32 flags);
extern int CommonArg(CommonState * state, int index);
extern const char *CommonUsage(CommonState * state);
extern SDL_bool CommonInit(CommonState * state);
extern void CommonEvent(CommonState * state, SDL_Event * event, int *done);
extern void CommonQuit(CommonState * state);