311 lines
19 KiB
Python
311 lines
19 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
Metadata about languages used by our model training code for our
|
|||
|
|
SingleByteCharSetProbers. Could be used for other things in the future.
|
|||
|
|
|
|||
|
|
This code is based on the language metadata from the uchardet project.
|
|||
|
|
"""
|
|||
|
|
from __future__ import absolute_import, print_function
|
|||
|
|
|
|||
|
|
from string import ascii_letters
|
|||
|
|
|
|||
|
|
|
|||
|
|
# TODO: Add Ukranian (KOI8-U)
|
|||
|
|
|
|||
|
|
class Language(object):
|
|||
|
|
"""Metadata about a language useful for training models
|
|||
|
|
|
|||
|
|
:ivar name: The human name for the language, in English.
|
|||
|
|
:type name: str
|
|||
|
|
:ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
|
|||
|
|
or use another catalog as a last resort.
|
|||
|
|
:type iso_code: str
|
|||
|
|
:ivar use_ascii: Whether or not ASCII letters should be included in trained
|
|||
|
|
models.
|
|||
|
|
:type use_ascii: bool
|
|||
|
|
:ivar charsets: The charsets we want to support and create data for.
|
|||
|
|
:type charsets: list of str
|
|||
|
|
:ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
|
|||
|
|
`True`, you only need to add those not in the ASCII set.
|
|||
|
|
:type alphabet: str
|
|||
|
|
:ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
|
|||
|
|
Wikipedia for training data.
|
|||
|
|
:type wiki_start_pages: list of str
|
|||
|
|
"""
|
|||
|
|
def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
|
|||
|
|
alphabet=None, wiki_start_pages=None):
|
|||
|
|
super(Language, self).__init__()
|
|||
|
|
self.name = name
|
|||
|
|
self.iso_code = iso_code
|
|||
|
|
self.use_ascii = use_ascii
|
|||
|
|
self.charsets = charsets
|
|||
|
|
if self.use_ascii:
|
|||
|
|
if alphabet:
|
|||
|
|
alphabet += ascii_letters
|
|||
|
|
else:
|
|||
|
|
alphabet = ascii_letters
|
|||
|
|
elif not alphabet:
|
|||
|
|
raise ValueError('Must supply alphabet if use_ascii is False')
|
|||
|
|
self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None
|
|||
|
|
self.wiki_start_pages = wiki_start_pages
|
|||
|
|
|
|||
|
|
def __repr__(self):
|
|||
|
|
return '{}({})'.format(self.__class__.__name__,
|
|||
|
|
', '.join('{}={!r}'.format(k, v)
|
|||
|
|
for k, v in self.__dict__.items()
|
|||
|
|
if not k.startswith('_')))
|
|||
|
|
|
|||
|
|
|
|||
|
|
LANGUAGES = {'Arabic': Language(name='Arabic',
|
|||
|
|
iso_code='ar',
|
|||
|
|
use_ascii=False,
|
|||
|
|
# We only support encodings that use isolated
|
|||
|
|
# forms, because the current recommendation is
|
|||
|
|
# that the rendering system handles presentation
|
|||
|
|
# forms. This means we purposefully skip IBM864.
|
|||
|
|
charsets=['ISO-8859-6', 'WINDOWS-1256',
|
|||
|
|
'CP720', 'CP864'],
|
|||
|
|
alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ',
|
|||
|
|
wiki_start_pages=[u'الصفحة_الرئيسية']),
|
|||
|
|
'Belarusian': Language(name='Belarusian',
|
|||
|
|
iso_code='be',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-5', 'WINDOWS-1251',
|
|||
|
|
'IBM866', 'MacCyrillic'],
|
|||
|
|
alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ'
|
|||
|
|
u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'),
|
|||
|
|
wiki_start_pages=[u'Галоўная_старонка']),
|
|||
|
|
'Bulgarian': Language(name='Bulgarian',
|
|||
|
|
iso_code='bg',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-5', 'WINDOWS-1251',
|
|||
|
|
'IBM855'],
|
|||
|
|
alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ'
|
|||
|
|
u'абвгдежзийклмнопрстуфхцчшщъьюя'),
|
|||
|
|
wiki_start_pages=[u'Начална_страница']),
|
|||
|
|
'Czech': Language(name='Czech',
|
|||
|
|
iso_code='cz',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ',
|
|||
|
|
wiki_start_pages=[u'Hlavní_strana']),
|
|||
|
|
'Danish': Language(name='Danish',
|
|||
|
|
iso_code='da',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'æøåÆØÅ',
|
|||
|
|
wiki_start_pages=[u'Forside']),
|
|||
|
|
'German': Language(name='German',
|
|||
|
|
iso_code='de',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'WINDOWS-1252'],
|
|||
|
|
alphabet=u'äöüßÄÖÜ',
|
|||
|
|
wiki_start_pages=[u'Wikipedia:Hauptseite']),
|
|||
|
|
'Greek': Language(name='Greek',
|
|||
|
|
iso_code='el',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-7', 'WINDOWS-1253'],
|
|||
|
|
alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ'
|
|||
|
|
u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'),
|
|||
|
|
wiki_start_pages=[u'Πύλη:Κύρια']),
|
|||
|
|
'English': Language(name='English',
|
|||
|
|
iso_code='en',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'WINDOWS-1252'],
|
|||
|
|
wiki_start_pages=[u'Main_Page']),
|
|||
|
|
'Esperanto': Language(name='Esperanto',
|
|||
|
|
iso_code='eo',
|
|||
|
|
# Q, W, X, and Y not used at all
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-3'],
|
|||
|
|
alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz'
|
|||
|
|
u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'),
|
|||
|
|
wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']),
|
|||
|
|
'Spanish': Language(name='Spanish',
|
|||
|
|
iso_code='es',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ',
|
|||
|
|
wiki_start_pages=[u'Wikipedia:Portada']),
|
|||
|
|
'Estonian': Language(name='Estonian',
|
|||
|
|
iso_code='et',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-4', 'ISO-8859-13',
|
|||
|
|
'WINDOWS-1257'],
|
|||
|
|
# C, F, Š, Q, W, X, Y, Z, Ž are only for
|
|||
|
|
# loanwords
|
|||
|
|
alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ'
|
|||
|
|
u'abdeghijklmnoprstuvõäöü'),
|
|||
|
|
wiki_start_pages=[u'Esileht']),
|
|||
|
|
'Finnish': Language(name='Finnish',
|
|||
|
|
iso_code='fi',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'ÅÄÖŠŽåäöšž',
|
|||
|
|
wiki_start_pages=[u'Wikipedia:Etusivu']),
|
|||
|
|
'French': Language(name='French',
|
|||
|
|
iso_code='fr',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ',
|
|||
|
|
wiki_start_pages=[u'Wikipédia:Accueil_principal',
|
|||
|
|
u'Bœuf (animal)']),
|
|||
|
|
'Hebrew': Language(name='Hebrew',
|
|||
|
|
iso_code='he',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-8', 'WINDOWS-1255'],
|
|||
|
|
alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ',
|
|||
|
|
wiki_start_pages=[u'עמוד_ראשי']),
|
|||
|
|
'Croatian': Language(name='Croatian',
|
|||
|
|
iso_code='hr',
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=(u'abcčćdđefghijklmnoprsštuvzž'
|
|||
|
|
u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'),
|
|||
|
|
wiki_start_pages=[u'Glavna_stranica']),
|
|||
|
|
'Hungarian': Language(name='Hungarian',
|
|||
|
|
iso_code='hu',
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű'
|
|||
|
|
u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'),
|
|||
|
|
wiki_start_pages=[u'Kezdőlap']),
|
|||
|
|
'Italian': Language(name='Italian',
|
|||
|
|
iso_code='it',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'ÀÈÉÌÒÓÙàèéìòóù',
|
|||
|
|
wiki_start_pages=[u'Pagina_principale']),
|
|||
|
|
'Lithuanian': Language(name='Lithuanian',
|
|||
|
|
iso_code='lt',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-13', 'WINDOWS-1257',
|
|||
|
|
'ISO-8859-4'],
|
|||
|
|
# Q, W, and X not used at all
|
|||
|
|
alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ'
|
|||
|
|
u'aąbcčdeęėfghiįyjklmnoprsštuųūvzž'),
|
|||
|
|
wiki_start_pages=[u'Pagrindinis_puslapis']),
|
|||
|
|
'Latvian': Language(name='Latvian',
|
|||
|
|
iso_code='lv',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-13', 'WINDOWS-1257',
|
|||
|
|
'ISO-8859-4'],
|
|||
|
|
# Q, W, X, Y are only for loanwords
|
|||
|
|
alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ'
|
|||
|
|
u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'),
|
|||
|
|
wiki_start_pages=[u'Sākumlapa']),
|
|||
|
|
'Macedonian': Language(name='Macedonian',
|
|||
|
|
iso_code='mk',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-5', 'WINDOWS-1251',
|
|||
|
|
'MacCyrillic', 'IBM855'],
|
|||
|
|
alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ'
|
|||
|
|
u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'),
|
|||
|
|
wiki_start_pages=[u'Главна_страница']),
|
|||
|
|
'Dutch': Language(name='Dutch',
|
|||
|
|
iso_code='nl',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'WINDOWS-1252'],
|
|||
|
|
wiki_start_pages=[u'Hoofdpagina']),
|
|||
|
|
'Polish': Language(name='Polish',
|
|||
|
|
iso_code='pl',
|
|||
|
|
# Q and X are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ'
|
|||
|
|
u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'),
|
|||
|
|
wiki_start_pages=[u'Wikipedia:Strona_główna']),
|
|||
|
|
'Portuguese': Language(name='Portuguese',
|
|||
|
|
iso_code='pt',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-1', 'ISO-8859-15',
|
|||
|
|
'WINDOWS-1252'],
|
|||
|
|
alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú',
|
|||
|
|
wiki_start_pages=[u'Wikipédia:Página_principal']),
|
|||
|
|
'Romanian': Language(name='Romanian',
|
|||
|
|
iso_code='ro',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=u'ăâîșțĂÂÎȘȚ',
|
|||
|
|
wiki_start_pages=[u'Pagina_principală']),
|
|||
|
|
'Russian': Language(name='Russian',
|
|||
|
|
iso_code='ru',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-5', 'WINDOWS-1251',
|
|||
|
|
'KOI8-R', 'MacCyrillic', 'IBM866',
|
|||
|
|
'IBM855'],
|
|||
|
|
alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
|
|||
|
|
u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
|
|||
|
|
wiki_start_pages=[u'Заглавная_страница']),
|
|||
|
|
'Slovak': Language(name='Slovak',
|
|||
|
|
iso_code='sk',
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ',
|
|||
|
|
wiki_start_pages=[u'Hlavná_stránka']),
|
|||
|
|
'Slovene': Language(name='Slovene',
|
|||
|
|
iso_code='sl',
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-2', 'WINDOWS-1250'],
|
|||
|
|
alphabet=(u'abcčdefghijklmnoprsštuvzž'
|
|||
|
|
u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'),
|
|||
|
|
wiki_start_pages=[u'Glavna_stran']),
|
|||
|
|
# Serbian can be written in both Latin and Cyrillic, but there's no
|
|||
|
|
# simple way to get the Latin alphabet pages from Wikipedia through
|
|||
|
|
# the API, so for now we just support Cyrillic.
|
|||
|
|
'Serbian': Language(name='Serbian',
|
|||
|
|
iso_code='sr',
|
|||
|
|
alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ'
|
|||
|
|
u'абвгдђежзијклљмнњопрстћуфхцчџш'),
|
|||
|
|
charsets=['ISO-8859-5', 'WINDOWS-1251',
|
|||
|
|
'MacCyrillic', 'IBM855'],
|
|||
|
|
wiki_start_pages=[u'Главна_страна']),
|
|||
|
|
'Thai': Language(name='Thai',
|
|||
|
|
iso_code='th',
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-11', 'TIS-620', 'CP874'],
|
|||
|
|
alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛',
|
|||
|
|
wiki_start_pages=[u'หน้าหลัก']),
|
|||
|
|
'Turkish': Language(name='Turkish',
|
|||
|
|
iso_code='tr',
|
|||
|
|
# Q, W, and X are not used by Turkish
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=['ISO-8859-3', 'ISO-8859-9',
|
|||
|
|
'WINDOWS-1254'],
|
|||
|
|
alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû'
|
|||
|
|
u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'),
|
|||
|
|
wiki_start_pages=[u'Ana_Sayfa']),
|
|||
|
|
'Vietnamese': Language(name='Vietnamese',
|
|||
|
|
iso_code='vi',
|
|||
|
|
use_ascii=False,
|
|||
|
|
# Windows-1258 is the only common 8-bit
|
|||
|
|
# Vietnamese encoding supported by Python.
|
|||
|
|
# From Wikipedia:
|
|||
|
|
# For systems that lack support for Unicode,
|
|||
|
|
# dozens of 8-bit Vietnamese code pages are
|
|||
|
|
# available.[1] The most common are VISCII
|
|||
|
|
# (TCVN 5712:1993), VPS, and Windows-1258.[3]
|
|||
|
|
# Where ASCII is required, such as when
|
|||
|
|
# ensuring readability in plain text e-mail,
|
|||
|
|
# Vietnamese letters are often encoded
|
|||
|
|
# according to Vietnamese Quoted-Readable
|
|||
|
|
# (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
|
|||
|
|
# though usage of either variable-width
|
|||
|
|
# scheme has declined dramatically following
|
|||
|
|
# the adoption of Unicode on the World Wide
|
|||
|
|
# Web.
|
|||
|
|
charsets=['WINDOWS-1258'],
|
|||
|
|
alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy'
|
|||
|
|
u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'),
|
|||
|
|
wiki_start_pages=[u'Chữ_Quốc_ngữ']),
|
|||
|
|
}
|