uchardet/src/LangModels/LangCzechModel.cpp
Jehan 26e1cebad1 LangModels: add support for Czech.
Encodings: Windows-1250, ISO-8859-2, IBM852 and Mac-CentralEurope.
Other encodings are known to have been used for Czech: Kamenicky,
KOI-8 CS2 and Cork. But these are uncommon enough that I decided not
to support them (especially since I can't find them supported in iconv
either, or at least not under an alias which I could recognize).
This web page, which contents was made under the Public Domain, is a
good reference for encodings which were used historically for Czech and
Slovak: http://luki.sdf-eu.org/txt/cs-encodings-faq.html
2016-09-21 03:33:50 +02:00

281 lines
12 KiB
C++

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is Mozilla Communicator client code.
*
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include "../nsSBCharSetProber.h"
/********* Language model for: Czech *********/
/**
* Generated by BuildLangModel.py
* On: 2016-09-21 03:28:11.733089
**/
/* Character Mapping Table:
* ILL: illegal character.
* CTR: control character specific to the charset.
* RET: carriage/return.
* SYM: symbol (punctuation) that does not belong to word.
* NUM: 0 - 9.
*
* Other characters are ordered by probabilities
* (0 is the most common character in the language).
*
* Orders are generic to a language. So the codepoint with order X in
* CHARSET1 maps to the same character as the codepoint with the same
* order X in CHARSET2 for the same language.
* As such, it is possible to get missing order. For instance the
* ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1
* even though they are both used for French. Same for the euro sign.
*/
static const unsigned char Windows_1250_CharToOrderMap[] =
{
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 4X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,SYM, /* 5X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 6X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,CTR, /* 7X */
SYM,ILL,SYM,ILL,SYM,SYM,SYM,SYM,ILL,SYM, 29,SYM, 46, 38, 26, 47, /* 8X */
ILL,SYM,SYM,SYM,SYM,SYM,SYM,SYM,ILL,SYM, 29,SYM, 46, 38, 26, 48, /* 9X */
SYM,SYM,SYM, 49,SYM, 50,SYM,SYM,SYM,SYM, 51,SYM,SYM,SYM,SYM, 52, /* AX */
SYM,SYM,SYM, 53,SYM,SYM,SYM,SYM,SYM, 54, 55,SYM, 45,SYM, 45, 56, /* BX */
57, 18, 58, 59, 42, 60, 61, 62, 25, 24, 63, 64, 23, 11, 65, 39, /* CX */
66, 67, 35, 37, 68, 69, 41,SYM, 27, 31, 33, 70, 43, 28, 71, 72, /* DX */
73, 18, 74, 75, 42, 76, 77, 78, 25, 24, 79, 80, 23, 11, 81, 39, /* EX */
82, 83, 35, 37, 84, 85, 41,SYM, 27, 31, 33, 86, 43, 28, 87,SYM, /* FX */
};
/*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
static const unsigned char Mac_Centraleurope_CharToOrderMap[] =
{
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 4X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,SYM, /* 5X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 6X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,CTR, /* 7X */
42, 88, 89, 24, 90, 41, 43, 18, 91, 25, 42, 25, 92, 93, 24, 94, /* 8X */
95, 39, 11, 39, 44, 44, 96, 37, 97, 98, 41, 99, 33, 23, 23, 43, /* 9X */
SYM,SYM,100,SYM,SYM,SYM,SYM,101,SYM,SYM,SYM,102,SYM,SYM,103,104, /* AX */
105,106,SYM,SYM,107,108,SYM,SYM,109,110,111, 45, 45,112,113,114, /* BX */
115,116,SYM,SYM,117, 35,SYM,SYM,SYM,SYM,SYM, 35,118,119,120,121, /* CX */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,122,123,124, 27,SYM,SYM, 27,125, /* DX */
126, 29,SYM,SYM, 29, 46, 46, 18, 38, 38, 11, 26, 26,127, 37,128, /* EX */
129, 31, 33, 31,130,131,132,133, 28, 28,134,135,136,137,138,SYM, /* FX */
};
/*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
static const unsigned char Ibm852_CharToOrderMap[] =
{
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 4X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,SYM, /* 5X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 6X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,CTR, /* 7X */
139, 43, 24,140, 42, 31,141,142,143,144,145,146,147,148, 42,149, /* 8X */
24,150,151,152, 41, 45, 45, 46, 46, 41, 43, 38, 38,153,SYM, 25, /* 9X */
18, 11, 37, 33,154,155, 26, 26,156,157,SYM,158, 25,159,SYM,SYM, /* AX */
SYM,SYM,SYM,SYM,SYM, 18,160, 23,161,SYM,SYM,SYM,SYM,162,163,SYM, /* BX */
SYM,SYM,SYM,SYM,SYM,SYM,164,165,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* CX */
166,167, 39,168, 39, 35, 11,169, 23,SYM,SYM,SYM,SYM,170, 31,SYM, /* DX */
37,171,172,173,174, 35, 29, 29,175, 33,176,177, 28, 28,178,SYM, /* EX */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,179, 27, 27,SYM,SYM, /* FX */
};
/*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
static const unsigned char Iso_8859_2_CharToOrderMap[] =
{
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 4X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,SYM, /* 5X */
SYM, 3, 22, 14, 15, 1, 30, 32, 17, 4, 21, 12, 10, 16, 2, 0, /* 6X */
8, 40, 9, 6, 5, 13, 7, 36, 34, 20, 19,SYM,SYM,SYM,SYM,CTR, /* 7X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 8X */
CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 9X */
SYM,180,SYM,181,SYM, 45, 46,SYM,SYM, 29,182, 38,183,SYM, 26,184, /* AX */
SYM,185,SYM,186,SYM, 45, 46,SYM,SYM, 29,187, 38,188,SYM, 26,189, /* BX */
190, 18,191,192, 42,193,194,195, 25, 24,196,197, 23, 11,198, 39, /* CX */
199,200, 35, 37,201,202, 41,SYM, 27, 31, 33,203, 43, 28,204,205, /* DX */
206, 18,207,208, 42,209,210,211, 25, 24,212,213, 23, 11,214, 39, /* EX */
215,216, 35, 37,217,218, 41,SYM, 27, 31, 33,219, 43, 28,220,SYM, /* FX */
};
/*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
/* Model Table:
* Total sequences: 1025
* First 512 sequences: 0.9786035192432675
* Next 512 sequences (512-1024): 0.02139445610866691
* Rest: 2.0246480655940202e-06
* Negative sequences: TODO
*/
static const PRUint8 CzechLangModel[] =
{
2,2,3,2,3,3,3,3,3,3,3,2,3,3,3,3,3,3,2,3,
2,3,3,0,0,3,3,3,0,2,3,0,3,0,3,2,2,0,2,0,0,
3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,2,3,
2,3,3,0,0,3,3,3,0,3,3,2,3,2,3,2,2,2,2,2,2,
3,3,3,3,3,3,3,2,0,2,3,3,3,3,3,3,2,3,3,3,
3,2,2,3,3,2,2,0,3,2,3,3,3,0,2,0,0,2,0,0,2,
3,3,3,2,2,3,3,3,3,3,3,0,3,3,3,3,3,3,0,3,
3,3,3,0,0,3,3,3,0,3,3,0,3,0,3,2,2,0,2,2,0,
3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,
0,2,3,0,2,3,3,2,0,3,3,0,3,0,2,2,2,2,2,0,2,
3,3,3,3,3,2,2,3,2,3,3,3,3,3,2,2,2,3,3,3,
3,2,2,3,3,2,0,3,3,3,0,3,2,0,0,2,2,2,0,0,0,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,2,
3,2,3,0,2,2,0,0,2,0,2,2,2,2,0,2,2,0,2,0,0,
3,3,3,3,3,2,2,0,2,3,3,3,3,3,2,3,0,2,3,3,
3,2,2,3,3,2,2,2,3,3,0,3,0,0,0,2,0,2,0,0,0,
3,3,3,3,3,3,3,0,2,3,3,3,2,3,2,2,2,2,3,0,
3,2,2,3,2,2,0,3,2,2,2,3,2,0,2,2,0,0,0,0,0,
3,3,3,3,3,3,3,3,3,2,2,2,3,3,2,3,3,2,3,3,
3,0,3,0,3,3,2,0,3,2,2,3,3,0,0,2,2,2,2,2,2,
3,3,3,3,3,3,2,2,2,2,2,3,3,3,2,2,2,2,3,3,
3,0,2,0,3,2,2,0,3,3,2,3,2,0,0,2,0,2,0,0,0,
0,2,3,0,2,3,3,3,3,3,3,2,3,0,3,2,3,3,0,3,
0,3,2,0,0,3,3,2,0,2,0,0,2,0,0,0,0,0,2,0,0,
3,3,3,3,3,3,2,3,0,3,3,0,2,3,3,3,2,2,3,2,
3,2,3,0,3,2,2,2,3,0,2,3,2,0,0,0,0,2,0,0,0,
2,2,3,3,3,3,3,3,3,3,3,0,3,2,3,3,3,3,3,3,
2,3,3,0,0,3,3,2,0,3,2,0,2,0,2,2,2,0,2,2,0,
3,3,3,3,3,3,2,2,2,2,2,3,3,2,2,3,2,3,2,2,
2,0,2,0,2,0,0,0,0,0,2,2,0,0,0,2,0,0,0,0,2,
3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,2,3,2,3,2,
3,0,2,3,3,2,2,2,2,2,2,3,2,0,2,2,2,0,0,0,0,
3,3,3,3,3,2,2,0,3,3,3,3,2,3,2,2,2,2,3,2,
3,2,3,3,3,2,3,2,2,2,2,3,2,0,0,2,0,2,0,0,0,
3,3,3,3,3,3,2,3,2,3,3,2,2,3,2,2,2,0,3,0,
3,2,2,0,2,2,2,2,3,0,2,2,0,0,0,0,2,2,2,0,0,
0,0,3,0,0,3,3,3,2,3,3,0,3,0,3,3,3,3,0,3,
0,2,2,0,0,2,3,2,0,3,2,0,0,0,0,2,0,0,0,2,0,
3,3,3,3,3,3,2,3,3,2,3,3,3,3,2,3,3,3,3,2,
3,2,2,0,2,2,0,2,2,2,2,2,0,2,0,2,0,2,0,0,0,
2,2,3,2,2,3,3,3,3,2,3,2,3,2,3,2,3,3,0,3,
0,2,3,0,0,2,3,2,0,3,2,0,2,2,2,0,0,0,2,0,0,
2,3,3,3,3,2,3,2,2,2,2,3,2,2,2,2,3,2,2,2,
0,2,2,0,0,2,0,0,0,3,2,2,0,2,0,2,0,2,0,2,0,
3,3,3,3,3,3,3,3,0,3,3,3,2,3,2,2,2,2,3,2,
3,3,2,3,2,2,0,2,3,2,0,2,0,0,0,2,0,2,0,0,0,
0,0,3,2,0,3,3,2,3,3,3,0,3,0,3,3,2,3,0,2,
0,3,0,0,0,2,3,3,0,3,0,0,0,0,0,2,0,0,2,2,0,
2,0,3,0,0,3,2,2,2,2,2,0,3,0,0,2,3,3,0,3,
0,0,0,0,0,3,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,
3,3,3,3,3,3,2,0,0,2,3,3,3,3,2,0,0,0,3,0,
0,0,3,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
3,3,3,2,3,2,2,0,0,2,2,3,2,2,2,3,2,0,3,0,
0,0,2,0,0,0,0,0,0,3,0,2,0,0,0,2,0,0,0,2,0,
2,3,2,3,3,0,2,0,0,0,0,3,2,2,0,0,0,0,2,2,
0,0,2,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
0,0,2,0,0,3,3,3,2,3,2,0,2,2,3,2,3,2,0,3,
0,2,3,0,0,2,2,2,0,3,2,0,0,0,0,0,0,0,0,0,0,
2,3,3,3,3,3,2,2,2,0,3,3,3,3,0,0,0,0,2,0,
0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,0,
3,3,2,3,3,2,2,0,2,3,3,2,0,3,0,2,2,0,2,2,
3,0,0,0,2,0,0,0,2,0,2,2,0,2,0,0,0,2,0,0,0,
0,0,2,2,0,0,3,3,0,2,2,0,2,0,2,2,3,2,0,3,
0,2,2,0,0,2,3,2,0,0,0,0,0,0,0,2,0,0,0,0,0,
3,3,3,3,3,2,2,2,2,3,3,0,0,2,2,2,2,2,2,0,
2,0,0,0,2,0,0,0,0,0,2,2,2,0,0,0,0,0,0,0,0,
0,0,2,0,0,2,3,2,2,2,2,0,2,0,2,2,2,2,0,3,
0,2,2,0,0,3,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,
2,2,2,2,3,3,0,0,3,2,2,2,2,2,2,2,2,0,2,0,
2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,2,0,0,0,0,
2,0,0,2,0,0,2,0,0,0,0,0,2,3,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
2,2,2,2,3,0,2,0,0,0,2,0,2,2,2,0,0,2,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,
0,0,2,0,0,0,2,0,0,0,2,0,0,0,0,2,2,0,0,3,
0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,
2,0,0,2,0,0,0,0,0,0,0,0,2,2,0,0,0,0,2,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
2,0,0,2,0,2,0,0,0,0,0,0,0,2,0,0,2,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
};
const SequenceModel Windows_1250CzechModel =
{
Windows_1250_CharToOrderMap,
CzechLangModel,
41,
(float)0.9786035192432675,
PR_TRUE,
"WINDOWS-1250"
};
const SequenceModel Mac_CentraleuropeCzechModel =
{
Mac_Centraleurope_CharToOrderMap,
CzechLangModel,
41,
(float)0.9786035192432675,
PR_TRUE,
"MAC-CENTRALEUROPE"
};
const SequenceModel Ibm852CzechModel =
{
Ibm852_CharToOrderMap,
CzechLangModel,
41,
(float)0.9786035192432675,
PR_TRUE,
"IBM852"
};
const SequenceModel Iso_8859_2CzechModel =
{
Iso_8859_2_CharToOrderMap,
CzechLangModel,
41,
(float)0.9786035192432675,
PR_TRUE,
"ISO-8859-2"
};