* enc/big5.c: split CP950 from Big5.

* enc/big5.c: split CP951 from Big5-HKSCS.

* enc/trans/big5.trans: import conversion table of Big5, Big5-HKSCS,
  CP950, and CP951 from ICU. they need fallback conversions.
  ref [ruby-core:33256]
  http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/

* tool/transcode-tblgen.rb (import_ucm): add to import ucm files.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@29869 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 21 insertions, 0 deletions

diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb
index edc8db72a9..833fc219a8 100755
--- a/tool/transcode-tblgen.rb
+++ b/tool/transcode-tblgen.rb
@@ -748,6 +748,27 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs)
   return table
 end
 
+def import_ucm(path)
+  to_ucs = []
+  from_ucs = []
+  File.foreach(File.join($srcdir, "ucm", path)) do |line|
+    uc, bs, fb = nil
+    if /^<U(\h+)>\s*([\+\hx\\]+)\s*\|(\d)/ =~ line
+      uc = $1.hex
+      bs = $2.delete('x\\')
+      fb = $3.to_i
+      next if uc < 128 && uc == bs.hex
+    elsif /^([<U\h>+]+)\s*([\+\hx\\]+)\s*\|(\d)/ =~ line
+      uc = $1.scan(/\h+>/).map(&:hex).pack("U*").unpack("H*")[0]
+      bs = $2.delete('x\\')
+      fb = $3.to_i
+    end
+    to_ucs << [bs, uc] if fb == 0 || fb == 3
+    from_ucs << [uc, bs] if fb == 0 || fb == 1
+  end
+  [to_ucs, from_ucs]
+end
+
 def encode_utf8(map)
   r = []
   map.each {|k, v|