Make unaccent handle all diacritics known to Unicode, and expand ligatures correctly

Add Python script for buiding unaccent.rules from Unicode data. Don't
backpatch because unaccent changes may require tsvector/index
rebuild.

Thomas Munro <[email protected]>

1 files changed, 292 insertions, 66 deletions

diff --git a/contrib/unaccent/unaccent.rules b/contrib/unaccent/unaccent.rules
index cc2f7a65858..73c24a188ba 100644
--- a/contrib/unaccent/unaccent.rules
+++ b/contrib/unaccent/unaccent.rules
@@ -4,22 +4,59 @@
 Ã	A
 Ä	A
 Å	A
-Æ	A
+Ç	C
+È	E
+É	E
+Ê	E
+Ë	E
+Ì	I
+Í	I
+Î	I
+Ï	I
+Ñ	N
+Ò	O
+Ó	O
+Ô	O
+Õ	O
+Ö	O
+Ù	U
+Ú	U
+Û	U
+Ü	U
+Ý	Y
 à	a
 á	a
 â	a
 ã	a
 ä	a
 å	a
-æ	a
+ç	c
+è	e
+é	e
+ê	e
+ë	e
+ì	i
+í	i
+î	i
+ï	i
+ñ	n
+ò	o
+ó	o
+ô	o
+õ	o
+ö	o
+ù	u
+ú	u
+û	u
+ü	u
+ý	y
+ÿ	y
 Ā	A
 ā	a
 Ă	A
 ă	a
 Ą	A
 ą	a
-Ç	C
-ç	c
 Ć	C
 ć	c
 Ĉ	C
@@ -30,16 +67,6 @@
 č	c
 Ď	D
 ď	d
-Đ	D
-đ	d
-È	E
-É	E
-Ê	E
-Ë	E
-è	e
-é	e
-ê	e
-ë	e
 Ē	E
 ē	e
 Ĕ	E
@@ -60,17 +87,7 @@
 ģ	g
 Ĥ	H
 ĥ	h
-Ħ	H
-ħ	h
 Ĩ	I
-Ì	I
-Í	I
-Î	I
-Ï	I
-ì	i
-í	i
-î	i
-ï	i
 ĩ	i
 Ī	I
 ī	i
@@ -79,62 +96,36 @@
 Į	I
 į	i
 İ	I
-ı	i
-IJ	I
-ij	i
+IJ	IJ
+ij	ij
 Ĵ	J
 ĵ	j
 Ķ	K
 ķ	k
-ĸ	k
 Ĺ	L
 ĺ	l
 Ļ	L
 ļ	l
 Ľ	L
 ľ	l
-Ŀ	L
-ŀ	l
-Ł	L
-ł	l
-Ñ	N
-ñ	n
 Ń	N
 ń	n
 Ņ	N
 ņ	n
 Ň	N
 ň	n
-ʼn	n
-Ŋ	N
-ŋ	n
-Ò	O
-Ó	O
-Ô	O
-Õ	O
-Ö	O
-ò	o
-ó	o
-ô	o
-õ	o
-ö	o
 Ō	O
 ō	o
 Ŏ	O
 ŏ	o
 Ő	O
 ő	o
-Œ	E
-œ	e
-Ø	O
-ø	o
 Ŕ	R
 ŕ	r
 Ŗ	R
 ŗ	r
 Ř	R
 ř	r
-ß	S
 Ś	S
 ś	s
 Ŝ	S
@@ -147,16 +138,6 @@
 ţ	t
 Ť	T
 ť	t
-Ŧ	T
-ŧ	t
-Ù	U
-Ú	U
-Û	U
-Ü	U
-ù	u
-ú	u
-û	u
-ü	u
 Ũ	U
 ũ	u
 Ū	U
@@ -171,9 +152,6 @@
 ų	u
 Ŵ	W
 ŵ	w
-Ý	Y
-ý	y
-ÿ	y
 Ŷ	Y
 ŷ	y
 Ÿ	Y
@@ -183,5 +161,253 @@
 ż	z
 Ž	Z
 ž	z
-ё	е
+Ơ	O
+ơ	o
+Ư	U
+ư	u
+DŽ	DZ
+Dž	Dz
+dž	dz
+LJ	LJ
+Lj	Lj
+lj	lj
+NJ	NJ
+Nj	Nj
+nj	nj
+Ǎ	A
+ǎ	a
+Ǐ	I
+ǐ	i
+Ǒ	O
+ǒ	o
+Ǔ	U
+ǔ	u
+Ǧ	G
+ǧ	g
+Ǩ	K
+ǩ	k
+Ǫ	O
+ǫ	o
+ǰ	j
+DZ	DZ
+Dz	Dz
+dz	dz
+Ǵ	G
+ǵ	g
+Ǹ	N
+ǹ	n
+Ȁ	A
+ȁ	a
+Ȃ	A
+ȃ	a
+Ȅ	E
+ȅ	e
+Ȇ	E
+ȇ	e
+Ȉ	I
+ȉ	i
+Ȋ	I
+ȋ	i
+Ȍ	O
+ȍ	o
+Ȏ	O
+ȏ	o
+Ȑ	R
+ȑ	r
+Ȓ	R
+ȓ	r
+Ȕ	U
+ȕ	u
+Ȗ	U
+ȗ	u
+Ș	S
+ș	s
+Ț	T
+ț	t
+Ȟ	H
+ȟ	h
+Ȧ	A
+ȧ	a
+Ȩ	E
+ȩ	e
+Ȯ	O
+ȯ	o
+Ȳ	Y
+ȳ	y
+Ḁ	A
+ḁ	a
+Ḃ	B
+ḃ	b
+Ḅ	B
+ḅ	b
+Ḇ	B
+ḇ	b
+Ḋ	D
+ḋ	d
+Ḍ	D
+ḍ	d
+Ḏ	D
+ḏ	d
+Ḑ	D
+ḑ	d
+Ḓ	D
+ḓ	d
+Ḙ	E
+ḙ	e
+Ḛ	E
+ḛ	e
+Ḟ	F
+ḟ	f
+Ḡ	G
+ḡ	g
+Ḣ	H
+ḣ	h
+Ḥ	H
+ḥ	h
+Ḧ	H
+ḧ	h
+Ḩ	H
+ḩ	h
+Ḫ	H
+ḫ	h
+Ḭ	I
+ḭ	i
+Ḱ	K
+ḱ	k
+Ḳ	K
+ḳ	k
+Ḵ	K
+ḵ	k
+Ḷ	L
+ḷ	l
+Ḻ	L
+ḻ	l
+Ḽ	L
+ḽ	l
+Ḿ	M
+ḿ	m
+Ṁ	M
+ṁ	m
+Ṃ	M
+ṃ	m
+Ṅ	N
+ṅ	n
+Ṇ	N
+ṇ	n
+Ṉ	N
+ṉ	n
+Ṋ	N
+ṋ	n
+Ṕ	P
+ṕ	p
+Ṗ	P
+ṗ	p
+Ṙ	R
+ṙ	r
+Ṛ	R
+ṛ	r
+Ṟ	R
+ṟ	r
+Ṡ	S
+ṡ	s
+Ṣ	S
+ṣ	s
+Ṫ	T
+ṫ	t
+Ṭ	T
+ṭ	t
+Ṯ	T
+ṯ	t
+Ṱ	T
+ṱ	t
+Ṳ	U
+ṳ	u
+Ṵ	U
+ṵ	u
+Ṷ	U
+ṷ	u
+Ṽ	V
+ṽ	v
+Ṿ	V
+ṿ	v
+Ẁ	W
+ẁ	w
+Ẃ	W
+ẃ	w
+Ẅ	W
+ẅ	w
+Ẇ	W
+ẇ	w
+Ẉ	W
+ẉ	w
+Ẋ	X
+ẋ	x
+Ẍ	X
+ẍ	x
+Ẏ	Y
+ẏ	y
+Ẑ	Z
+ẑ	z
+Ẓ	Z
+ẓ	z
+Ẕ	Z
+ẕ	z
+ẖ	h
+ẗ	t
+ẘ	w
+ẙ	y
+Ạ	A
+ạ	a
+Ả	A
+ả	a
+Ẹ	E
+ẹ	e
+Ẻ	E
+ẻ	e
+Ẽ	E
+ẽ	e
+Ỉ	I
+ỉ	i
+Ị	I
+ị	i
+Ọ	O
+ọ	o
+Ỏ	O
+ỏ	o
+Ụ	U
+ụ	u
+Ủ	U
+ủ	u
+Ỳ	Y
+ỳ	y
+Ỵ	Y
+ỵ	y
+Ỷ	Y
+ỷ	y
+Ỹ	Y
+ỹ	y
+ff	ff
+fi	fi
+fl	fl
+ffi	ffi
+ffl	ffl
+st	st
+Ø	O
+ø	o
+Đ	D
+đ	d
+ı	i
+Ħ	H
+ħ	h
+Ł	L
+ł	l
+ʼn	'n
+Ŧ	T
+ŧ	t
 Ё	Е
+ё	е
+Æ	AE
+ß	ss
+æ	ae
+Œ	OE
+œ	oe