coffee/uchardet
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/uchardet/uchardet.git synced 2025-12-07 17:26:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

nsSBCharSetProber: replace the fixed 64 SAMPLE_SIZE...

Browse Source 
... with per-language model "frequent character" count.
This commit is contained in:
Jehan 2015-11-29 23:51:55 +01:00
parent b64831ff89
commit dbb4c1d2ff
9 changed files with 27 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/LangModels/LangBulgarianModel.cpp
View File

@ -230,6 +230,7 @@ const SequenceModel Latin5BulgarianModel =
{
Latin5_BulgarianCharToOrderMap,
BulgarianLangModel,
64,
(float)0.969392,
PR_FALSE,
"ISO-8859-5"
@ -239,6 +240,7 @@ const SequenceModel Win1251BulgarianModel =
{
win1251BulgarianCharToOrderMap,
BulgarianLangModel,
64,
(float)0.969392,
PR_FALSE,
"WINDOWS-1251"

6
src/LangModels/LangCyrillicModel.cpp
View File

@ -304,6 +304,7 @@ const SequenceModel Koi8rCyrillicModel =
{
KOI8R_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"KOI8-R"
@ -313,6 +314,7 @@ const SequenceModel Win1251CyrillicModel =
{
win1251_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"WINDOWS-1251"
@ -322,6 +324,7 @@ const SequenceModel Latin5CyrillicModel =
{
latin5_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"ISO-8859-5"
@ -331,6 +334,7 @@ const SequenceModel MacCyrillicModel =
{
macCyrillic_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"MAC-CYRILLIC"
@ -340,6 +344,7 @@ const SequenceModel Ibm866CyrillicModel =
{
IBM866_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"IBM866"
@ -349,6 +354,7 @@ const SequenceModel Ibm855CyrillicModel =
{
IBM855_CharToOrderMap,
RussianLangModel,
64,
(float)0.976601,
PR_FALSE,
"IBM855"

2
src/LangModels/LangFrenchModel.cpp
View File

@ -229,6 +229,7 @@ const SequenceModel Iso_8859_15FrenchModel =
{
Iso_8859_15_CharToOrderMap,
FrenchLangModel,
64,
(float)0.98,
PR_TRUE,
"ISO-8859-15"
@ -238,6 +239,7 @@ const SequenceModel Iso_8859_1FrenchModel =
{
Iso_8859_1_CharToOrderMap,
FrenchLangModel,
64,
(float)0.99,
PR_TRUE,
"ISO-8859-1"

2
src/LangModels/LangGreekModel.cpp
View File

@ -229,6 +229,7 @@ const SequenceModel Latin7GreekModel =
{
Latin7_CharToOrderMap,
GreekLangModel,
64,
(float)0.982851,
PR_FALSE,
"ISO-8859-7"
@ -238,6 +239,7 @@ const SequenceModel Win1253GreekModel =
{
win1253_CharToOrderMap,
GreekLangModel,
64,
(float)0.982851,
PR_FALSE,
"WINDOWS-1253"

1
src/LangModels/LangHebrewModel.cpp
View File

@ -212,6 +212,7 @@ const SequenceModel Win1255Model =
{
win1255_CharToOrderMap,
HebrewLangModel,
64,
(float)0.984004,
PR_FALSE,
"WINDOWS-1255"

2
src/LangModels/LangHungarianModel.cpp
View File

@ -227,6 +227,7 @@ const SequenceModel Latin2HungarianModel =
{
Latin2_HungarianCharToOrderMap,
HungarianLangModel,
64,
(float)0.947368,
PR_TRUE,
"ISO-8859-2"
@ -236,6 +237,7 @@ const SequenceModel Win1250HungarianModel =
{
win1250HungarianCharToOrderMap,
HungarianLangModel,
64,
(float)0.947368,
PR_TRUE,
"WINDOWS-1250"

1
src/LangModels/LangThaiModel.cpp
View File

@ -215,6 +215,7 @@ const SequenceModel TIS620ThaiModel =
{
TIS620CharToOrderMap,
ThaiLangModel,
64,
(float)0.926386,
PR_FALSE,
"TIS-620"

8
src/nsSBCharSetProber.cpp
View File

@ -48,17 +48,17 @@ nsProbingState nsSingleByteCharSetProber::HandleData(const char* aBuf, PRUint32
if (order < SYMBOL_CAT_ORDER)
mTotalChar++;
if (order < SAMPLE_SIZE)
if (order < mModel->freqCharCount)
{
mFreqChar++;
if (mLastOrder < SAMPLE_SIZE)
if (mLastOrder < mModel->freqCharCount)
{
mTotalSeqs++;
if (!mReversed)
++(mSeqCounters[mModel->precedenceMatrix[mLastOrder*SAMPLE_SIZE+order]]);
++(mSeqCounters[mModel->precedenceMatrix[mLastOrder*mModel->freqCharCount+order]]);
else // reverse the order of the letters in the lookup
++(mSeqCounters[mModel->precedenceMatrix[order*SAMPLE_SIZE+mLastOrder]]);
++(mSeqCounters[mModel->precedenceMatrix[order*mModel->freqCharCount+mLastOrder]]);
}
}
mLastOrder = order;

11
src/nsSBCharSetProber.h
View File

@ -53,7 +53,6 @@
/* Numbers 0-9. */
#define NUM 251
#define SAMPLE_SIZE 64
#define SB_ENOUGH_REL_THRESHOLD 1024
#define POSITIVE_SHORTCUT_THRESHOLD (float)0.95
#define NEGATIVE_SHORTCUT_THRESHOLD (float)0.05
@ -64,9 +63,13 @@
typedef struct
{
const unsigned char* const charToOrderMap; // [256] table use to find a char's order
const PRUint8* const precedenceMatrix; // [SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency
float mTypicalPositiveRatio; // = freqSeqs / totalSeqs
/* [256] table mapping codepoints to chararacter orders. */
const unsigned char* const charToOrderMap;
/* freqCharCount x freqCharCount table of 2-char sequence's frequencies. */
const PRUint8* const precedenceMatrix;
/* The count of frequent characters. */
int freqCharCount;
float mTypicalPositiveRatio; // = freqSeqs / totalSeqs
PRBool keepEnglishLetter; // says if this script contains English characters (not implemented)
const char* const charsetName;
} SequenceModel;