|
135
|
1 # encoding.py - character transcoding support for Mercurial
|
|
|
2 #
|
|
|
3 # Copyright 2005-2009 Matt Mackall <mpm@selenic.com> and others
|
|
|
4 #
|
|
|
5 # This software may be used and distributed according to the terms of the
|
|
|
6 # GNU General Public License version 2, incorporated herein by reference.
|
|
|
7
|
|
|
8 import error
|
|
|
9 import sys, unicodedata, locale, os
|
|
|
10
|
|
|
11 _encodingfixup = {'646': 'ascii', 'ANSI_X3.4-1968': 'ascii'}
|
|
|
12
|
|
|
13 try:
|
|
|
14 encoding = os.environ.get("HGENCODING")
|
|
|
15 if sys.platform == 'darwin' and not encoding:
|
|
|
16 # On darwin, getpreferredencoding ignores the locale environment and
|
|
|
17 # always returns mac-roman. We override this if the environment is
|
|
|
18 # not C (has been customized by the user).
|
|
|
19 locale.setlocale(locale.LC_CTYPE, '')
|
|
|
20 encoding = locale.getlocale()[1]
|
|
|
21 if not encoding:
|
|
|
22 encoding = locale.getpreferredencoding() or 'ascii'
|
|
|
23 encoding = _encodingfixup.get(encoding, encoding)
|
|
|
24 except locale.Error:
|
|
|
25 encoding = 'ascii'
|
|
|
26 encodingmode = os.environ.get("HGENCODINGMODE", "strict")
|
|
|
27 fallbackencoding = 'ISO-8859-1'
|
|
|
28
|
|
|
29 def tolocal(s):
|
|
|
30 """
|
|
|
31 Convert a string from internal UTF-8 to local encoding
|
|
|
32
|
|
|
33 All internal strings should be UTF-8 but some repos before the
|
|
|
34 implementation of locale support may contain latin1 or possibly
|
|
|
35 other character sets. We attempt to decode everything strictly
|
|
|
36 using UTF-8, then Latin-1, and failing that, we use UTF-8 and
|
|
|
37 replace unknown characters.
|
|
|
38 """
|
|
|
39 for e in ('UTF-8', fallbackencoding):
|
|
|
40 try:
|
|
|
41 u = s.decode(e) # attempt strict decoding
|
|
|
42 return u.encode(encoding, "replace")
|
|
|
43 except LookupError, k:
|
|
|
44 raise error.Abort("%s, please check your locale settings" % k)
|
|
|
45 except UnicodeDecodeError:
|
|
|
46 pass
|
|
|
47 u = s.decode("utf-8", "replace") # last ditch
|
|
|
48 return u.encode(encoding, "replace")
|
|
|
49
|
|
|
50 def fromlocal(s):
|
|
|
51 """
|
|
|
52 Convert a string from the local character encoding to UTF-8
|
|
|
53
|
|
|
54 We attempt to decode strings using the encoding mode set by
|
|
|
55 HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown
|
|
|
56 characters will cause an error message. Other modes include
|
|
|
57 'replace', which replaces unknown characters with a special
|
|
|
58 Unicode character, and 'ignore', which drops the character.
|
|
|
59 """
|
|
|
60 try:
|
|
|
61 return s.decode(encoding, encodingmode).encode("utf-8")
|
|
|
62 except UnicodeDecodeError, inst:
|
|
|
63 sub = s[max(0, inst.start-10):inst.start+10]
|
|
|
64 raise error.Abort("decoding near '%s': %s!" % (sub, inst))
|
|
|
65 except LookupError, k:
|
|
|
66 raise error.Abort("%s, please check your locale settings" % k)
|
|
|
67
|
|
|
68 def colwidth(s):
|
|
|
69 "Find the column width of a UTF-8 string for display"
|
|
|
70 d = s.decode(encoding, 'replace')
|
|
|
71 if hasattr(unicodedata, 'east_asian_width'):
|
|
|
72 w = unicodedata.east_asian_width
|
|
|
73 return sum([w(c) in 'WF' and 2 or 1 for c in d])
|
|
|
74 return len(d)
|
|
|
75
|
