Fixed bug #896 John Popplewell 2009-12-08 23:05:50 PST Originally reported by AKFoerster on the mailing list. Error decoding UTF8 Russian text to UTF-16LE on Windows, but specifically on platforms without iconv support (the default on Windows). Valid UTF8 characters are flagged as being overlong and then substituted by the UNKNOWN_UNICODE character. After studying the testiconv.c example program, reading the RFCs and putting some printf statements in SDL_iconv.c the problem is in a test for 'Maximum overlong sequences', specifically 4.2.1, which is carried out by the following code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xCE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { Here is the 2-byte encoding of a character in range 00000080 - 000007FF 110xxxxx 10xxxxxx The line in question is supposed to be checking for an overlong sequence which would be less than 11000001 10111111 which should be represented as a single byte. BUT, the mask value (0xCE) is wrong, it isn't checking the top-most bit: 11000001 value 11001110 mask (incorrect) ^ and should be (0xDE): 11000001 value 11011110 mask (correct) making the above code: } else if ( p[0] >= 0xC0 ) { if ( (p[0] & 0xE0) != 0xC0 ) { /* Skip illegal sequences return SDL_ICONV_EILSEQ; */ ch = UNKNOWN_UNICODE; } else { if ( (p[0] & 0xDE) == 0xC0 ) { <<<<<<<< here overlong = SDL_TRUE; } ch = (Uint32)(p[0] & 0x1F); left = 1; } } else { I can supply a test program and/or a patch if required, best regards, John Popplewell

#! /bin/sh
# mkinstalldirs --- make directory hierarchy
# Author: Noah Friedman <friedman@prep.ai.mit.edu>
# Created: 1993-05-16
# Public domain

errstatus=0
dirmode=""

usage="\
Usage: mkinstalldirs [-h] [--help] [-m mode] dir ..."

# process command line arguments
while test $# -gt 0 ; do
   case "${1}" in
     -h | --help | --h* )			# -h for help
	echo "${usage}" 1>&2; exit 0 ;;
     -m )					# -m PERM arg
	shift
	test $# -eq 0 && { echo "${usage}" 1>&2; exit 1; }
	dirmode="${1}"
	shift ;;
     -- ) shift; break ;;			# stop option processing
     -* ) echo "${usage}" 1>&2; exit 1 ;;	# unknown option
     * )  break ;;				# first non-opt arg
   esac
done

for file
do
  if test -d "$file"; then
    shift
  else
    break
  fi
done

case $# in
0) exit 0 ;;
esac

case $dirmode in
'')
  if mkdir -p -- . 2>/dev/null; then
    echo "mkdir -p -- $*"
    exec mkdir -p -- "$@"
  fi ;;
*)
  if mkdir -m "$dirmode" -p -- . 2>/dev/null; then
    echo "mkdir -m $dirmode -p -- $*"
    exec mkdir -m "$dirmode" -p -- "$@"
  fi ;;
esac

for file
do
   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
   shift

   pathcomp=
   for d
   do
     pathcomp="$pathcomp$d"
     case "$pathcomp" in
       -* ) pathcomp=./$pathcomp ;;
     esac

     if test ! -d "$pathcomp"; then
	echo "mkdir $pathcomp"

	mkdir "$pathcomp" || lasterr=$?

	if test ! -d "$pathcomp"; then
	  errstatus=$lasterr
	else
	  if test ! -z "$dirmode"; then
	     echo "chmod $dirmode $pathcomp"

	     lasterr=""
	     chmod "$dirmode" "$pathcomp" || lasterr=$?

	     if test ! -z "$lasterr"; then
	       errstatus=$lasterr
	     fi
	  fi
	fi
     fi

     pathcomp="$pathcomp/"
   done
done

exit $errstatus

# Local Variables:
# mode: shell-script
# sh-indentation: 3
# End:
# mkinstalldirs ends here