mirror of
https://gitlab.freedesktop.org/uchardet/uchardet.git
synced 2025-12-06 16:56:40 +08:00
db836fad63
Right now, each time we add new language or new charset support, we have too many pieces of code not to forget to edit. The script script/BuildLangModel.py will now take care of the main parts: listing the sequence models, listing the generic language models and computing the numbers for each listing. Furthermore the script will now end with a TODO list of the parts which are still to be done manually (2 functions to edit and a CMakeLists). Finally the script now allows to give a list of languages to edit rather of having to run it with languages one by one. It also allows 2 special code: "none", which will retrain none of the languages, but will re-generate only the new generated listings; and "all" which will retrain all models (useful in particulare when we change the model formats or usage and want to regenerate everything).
888 lines
36 KiB
Python
Executable File
888 lines
36 KiB
Python
Executable File
#!/bin/python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# ##### BEGIN LICENSE BLOCK #####
|
|
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
|
#
|
|
# The contents of this file are subject to the Mozilla Public License Version
|
|
# 1.1 (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
# http://www.mozilla.org/MPL/
|
|
#
|
|
# Software distributed under the License is distributed on an "AS IS" basis,
|
|
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
|
# for the specific language governing rights and limitations under the
|
|
# License.
|
|
#
|
|
# The Original Code is Mozilla Universal charset detector code.
|
|
#
|
|
# The Initial Developer of the Original Code is
|
|
# Netscape Communications Corporation.
|
|
# Portions created by the Initial Developer are Copyright (C) 2001
|
|
# the Initial Developer. All Rights Reserved.
|
|
#
|
|
# Contributor(s):
|
|
# Jehan <jehan@girinstud.io>
|
|
#
|
|
# Alternatively, the contents of this file may be used under the terms of
|
|
# either the GNU General Public License Version 2 or later (the "GPL"), or
|
|
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
# in which case the provisions of the GPL or the LGPL are applicable instead
|
|
# of those above. If you wish to allow use of your version of this file only
|
|
# under the terms of either the GPL or the LGPL, and not to allow others to
|
|
# use your version of this file under the terms of the MPL, indicate your
|
|
# decision by deleting the provisions above and replace them with the notice
|
|
# and other provisions required by the GPL or the LGPL. If you do not delete
|
|
# the provisions above, a recipient may use your version of this file under
|
|
# the terms of any one of the MPL, the GPL or the LGPL.
|
|
#
|
|
# ##### END LICENSE BLOCK #####
|
|
|
|
# Third party modules.
|
|
import unicodedata
|
|
import subprocess
|
|
import wikipedia
|
|
import importlib
|
|
import math
|
|
import optparse
|
|
import datetime
|
|
import operator
|
|
import requests
|
|
import sys
|
|
import re
|
|
import os
|
|
import random
|
|
|
|
# Custom modules.
|
|
import charsets.db
|
|
from charsets.codepoints import *
|
|
|
|
# Command line processing.
|
|
usage = 'Usage: {} <LANG-CODE>\n' \
|
|
'\nEx: `{} fr`'.format(__file__, __file__)
|
|
|
|
description = "Internal tool for uchardet to generate language data."
|
|
cmdline = optparse.OptionParser(usage, description = description)
|
|
cmdline.add_option('--max-page',
|
|
help = 'Maximum number of Wikipedia pages to parse (useful for debugging).',
|
|
action = 'store', type = 'int', dest = 'max_page', default = None)
|
|
cmdline.add_option('--max-depth',
|
|
help = 'Maximum depth when following links from start page (default: 2).',
|
|
action = 'store', type = 'int',
|
|
dest = 'max_depth', default = 2)
|
|
(options, langs) = cmdline.parse_args()
|
|
if len(langs) < 1:
|
|
sys.stderr.write("Please select at least one language code. ")
|
|
sys.stderr.write("You may also choose 'all' or 'none'.\n")
|
|
exit(1)
|
|
|
|
current_dir = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
with open(os.path.join(current_dir, "support.txt")) as f:
|
|
all_langs = f.readlines()
|
|
all_langs = [ l.strip() for l in all_langs if l.strip() != '' ]
|
|
|
|
if len(langs) == 1:
|
|
if langs[0].lower() == 'none':
|
|
langs = []
|
|
elif langs[0].lower() == 'all':
|
|
langs = all_langs
|
|
|
|
abort = False
|
|
for lang in langs:
|
|
if lang not in all_langs:
|
|
abort = True
|
|
sys.stderr.write("Error: unsupported lang: {}\n".format(lang))
|
|
if abort:
|
|
sys.stderr.write("Info: new langs must be added in 'script/support.txt'.\n")
|
|
exit(1)
|
|
|
|
generated_files = []
|
|
|
|
for lang_arg in langs:
|
|
lang_arg = lang_arg.lower()
|
|
|
|
# Load the language data.
|
|
sys_path_backup = sys.path
|
|
sys.path = [current_dir + '/langs']
|
|
try:
|
|
lang = importlib.import_module(lang_arg)
|
|
except ImportError:
|
|
sys.stderr.write('Unknown language code "{}": '
|
|
'file "langs/{}.py" does not exist.'.format(lang_arg, lang_arg))
|
|
exit(1)
|
|
sys.path = sys_path_backup
|
|
|
|
print("Processing language data for {} (lang/{}.py):\n".format(lang_arg, lang_arg))
|
|
|
|
lang_charsets = charsets.db.load(lang.charsets)
|
|
|
|
if not hasattr(lang, 'start_pages') or lang.start_pages is None or \
|
|
lang.start_pages == []:
|
|
# Let's start with the main page, assuming it should have links
|
|
# to relevant pages. In locale wikipedia, this page is usually redirected
|
|
# to a relevant page.
|
|
sys.stderr.write("Warning: no `start_pages` set for '{}'. Using ['Main_Page'].\n"
|
|
" If you don't get good data, it is advised to set a "
|
|
"start_pages` variable yourself.".format(lang.code))
|
|
lang.start_pages = ['Main_Page']
|
|
if not hasattr(lang, 'wikipedia_code') or lang.wikipedia_code is None:
|
|
lang.wikipedia_code = lang.code
|
|
if not hasattr(lang, 'clean_wikipedia_content') or lang.clean_wikipedia_content is None:
|
|
lang.clean_wikipedia_content = None
|
|
if hasattr(lang, 'case_mapping'):
|
|
lang.case_mapping = bool(lang.case_mapping)
|
|
else:
|
|
lang.case_mapping = False
|
|
if not hasattr(lang, 'custom_case_mapping'):
|
|
lang.custom_case_mapping = None
|
|
if not hasattr(lang, 'alphabet') or lang.alphabet is None:
|
|
lang.alphabet = None
|
|
if not hasattr(lang, 'alphabet_mapping') or lang.alphabet_mapping is None:
|
|
lang.alphabet_mapping = None
|
|
if not hasattr(lang, 'unicode_ranges') or lang.unicode_ranges is None:
|
|
lang.unicode_ranges = None
|
|
if not hasattr(lang, 'frequent_ranges') or lang.frequent_ranges is None:
|
|
if lang.unicode_ranges is not None:
|
|
lang.frequent_ranges = lang.unicode_ranges
|
|
else:
|
|
lang.frequent_ranges = None
|
|
|
|
def local_lowercase(text, lang):
|
|
lowercased = ''
|
|
for l in text:
|
|
if lang.custom_case_mapping is not None and \
|
|
l in lang.custom_case_mapping:
|
|
lowercased += lang.custom_case_mapping[l]
|
|
elif l.isupper() and \
|
|
lang.case_mapping and \
|
|
len(unicodedata.normalize('NFC', l.lower())) == 1:
|
|
lowercased += l.lower()
|
|
else:
|
|
lowercased += l
|
|
return lowercased
|
|
|
|
if lang.use_ascii:
|
|
if lang.alphabet is None:
|
|
lang.alphabet = [chr(l) for l in range(65, 91)] + [chr(l) for l in range(97, 123)]
|
|
else:
|
|
# Allowing to provide an alphabet in string format rather than list.
|
|
lang.alphabet = list(lang.alphabet)
|
|
lang.alphabet += [chr(l) for l in range(65, 91)] + [chr(l) for l in range(97, 123)]
|
|
if lang.alphabet is not None:
|
|
# Allowing to provide an alphabet in string format rather than list.
|
|
lang.alphabet = list(lang.alphabet)
|
|
if lang.case_mapping or lang.custom_case_mapping is not None:
|
|
lang.alphabet = [local_lowercase(l, lang) for l in lang.alphabet]
|
|
#alphabet = []
|
|
#for l in lang.alphabet:
|
|
#if l.isupper() and \
|
|
#lang.custom_case_mapping is not None and \
|
|
#l in lang.custom_case_mapping:
|
|
#alphabet.append(lang.custom_case_mapping[l])
|
|
#elif l.isupper() and \
|
|
#lang.case_mapping and \
|
|
#len(unicodedata.normalize('NFC', l.lower())) == 1:
|
|
#alphabet.append(l.lower())
|
|
#else:
|
|
#alphabet.append(l)
|
|
lang.alphabet = list(set(lang.alphabet))
|
|
|
|
if lang.alphabet_mapping is not None:
|
|
alphabet_mapping = {}
|
|
for char in lang.alphabet_mapping:
|
|
# Allowing to provide an alphabet in string format rather than list.
|
|
for alt_char in list(lang.alphabet_mapping[char]):
|
|
# While it's easier to write from main character to
|
|
# equivalencies in the language file, we reverse the mapping
|
|
# for simpler usage.
|
|
if lang.case_mapping or lang.custom_case_mapping is not None:
|
|
alphabet_mapping[alt_char] = local_lowercase(char, lang)
|
|
else:
|
|
alphabet_mapping[alt_char] = char
|
|
lang.alphabet_mapping = alphabet_mapping
|
|
|
|
def normalize_codepoint_ranges(input_range):
|
|
output_range = []
|
|
if input_range is not None:
|
|
for start, end in input_range:
|
|
# Allow to write down characters rather than unicode values.
|
|
if isinstance(start, str):
|
|
start = ord(start)
|
|
if isinstance(end, str):
|
|
end = ord(end)
|
|
if not isinstance(start, int) or not isinstance(end, int):
|
|
sys.stderr.write("Expected unicode range in char or int: {}-{}.\n".format(start, end))
|
|
if start > end:
|
|
sys.stderr.write("Wrong unicode range: {}-{}.\n".format(start, end))
|
|
else:
|
|
output_range += [(start, end)]
|
|
if len(output_range) == 0:
|
|
output_range = None
|
|
return output_range
|
|
|
|
lang.unicode_ranges = normalize_codepoint_ranges(lang.unicode_ranges)
|
|
lang.frequent_ranges = normalize_codepoint_ranges(lang.frequent_ranges)
|
|
|
|
# Starting processing.
|
|
wikipedia.set_lang(lang.wikipedia_code)
|
|
|
|
visited_pages = []
|
|
|
|
# The full list of letter characters.
|
|
# The key is the unicode codepoint,
|
|
# and the value is the occurrence count.
|
|
characters = {}
|
|
# Sequence of letters.
|
|
# The key is the couple (char1, char2) in unicode codepoint,
|
|
# the value is the occurrence count.
|
|
sequences = {}
|
|
prev_char = None
|
|
|
|
def process_text(content, lang):
|
|
global lang_charsets
|
|
global characters
|
|
global sequences
|
|
global prev_char
|
|
|
|
if lang.clean_wikipedia_content is not None:
|
|
content = lang.clean_wikipedia_content(content)
|
|
# Clean out the Wikipedia syntax for titles.
|
|
content = re.sub(r'(=+) *([^=]+) *\1',
|
|
r'\2', content)
|
|
# Clean multiple spaces. Newlines and such are normalized to spaces,
|
|
# since they have basically a similar role in the purpose of uchardet.
|
|
content = re.sub(r'\s+', ' ', content)
|
|
|
|
if lang.case_mapping or lang.custom_case_mapping is not None:
|
|
content = local_lowercase(content, lang)
|
|
|
|
# In python 3, strings are UTF-8.
|
|
# Looping through them return expected characters.
|
|
for char in content:
|
|
# Map to main equivalent character.
|
|
if lang.alphabet_mapping is not None and \
|
|
char in lang.alphabet_mapping:
|
|
char = lang.alphabet_mapping[char]
|
|
|
|
unicode_value = ord(char)
|
|
is_letter = False
|
|
if unicode_value in characters:
|
|
characters[unicode_value] += 1
|
|
is_letter = True
|
|
elif lang.unicode_ranges is not None:
|
|
for start, end in lang.unicode_ranges:
|
|
if unicode_value >= start and unicode_value <= end:
|
|
characters[unicode_value] = 1
|
|
is_letter = True
|
|
break
|
|
else:
|
|
# We save the character if it is at least in one of the
|
|
# language encodings and its not a special character.
|
|
for charset in lang_charsets:
|
|
# Does the character exist in the charset?
|
|
try:
|
|
codepoint = char.encode(charset, 'ignore')
|
|
except LookupError:
|
|
# unknown encoding. Use iconv from command line instead.
|
|
try:
|
|
call = subprocess.Popen(['iconv', '-f', 'UTF-8', '-t', charset],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL)
|
|
if call.poll() is not None:
|
|
(_, error) = call.communicate(input='')
|
|
sys.stderr.write('Error: `iconv` ended with error "{}".\n'.format(error))
|
|
exit(1)
|
|
(codepoint, _) = call.communicate(input=char.encode('UTF-8'))
|
|
except FileNotFoundError:
|
|
sys.stderr.write('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
|
|
exit(1)
|
|
|
|
if codepoint == b'':
|
|
continue
|
|
# ord() is said to return the unicode codepoint.
|
|
# But it turns out it also gives the codepoint for other
|
|
# charsets if I turn the string to encoded bytes first.
|
|
# Not sure if that is a bug or expected.
|
|
codepoint = ord(codepoint)
|
|
if lang_charsets[charset].charmap[codepoint] == LET:
|
|
characters[unicode_value] = 1
|
|
is_letter = True
|
|
break
|
|
if is_letter:
|
|
if prev_char is not None:
|
|
if (prev_char, unicode_value) in sequences:
|
|
sequences[(prev_char, unicode_value)] += 1
|
|
else:
|
|
sequences[(prev_char, unicode_value)] = 1
|
|
prev_char = unicode_value
|
|
else:
|
|
prev_char = None
|
|
|
|
def visit_pages(titles, depth, lang, logfd):
|
|
global visited_pages
|
|
global options
|
|
|
|
if len(titles) == 0:
|
|
return
|
|
|
|
next_titles = []
|
|
if options.max_page is not None:
|
|
max_titles = int(options.max_page/(options.max_depth * options.max_depth))
|
|
else:
|
|
max_titles = sys.maxsize
|
|
for title in titles:
|
|
if options.max_page is not None and \
|
|
len(visited_pages) > options.max_page:
|
|
return
|
|
if title in visited_pages:
|
|
continue
|
|
|
|
# Ugly hack skipping internal pages
|
|
if 'wiki' in title or 'Wiki' in title:
|
|
sys.stderr.write('Skipping', title)
|
|
continue
|
|
|
|
visited_pages += [title]
|
|
try:
|
|
page = wikipedia.page(title, auto_suggest=False)
|
|
except (wikipedia.exceptions.PageError,
|
|
wikipedia.exceptions.DisambiguationError) as error:
|
|
# Let's just discard a page when I get an exception.
|
|
sys.stderr.write("Discarding page {}: {}\n".format(title, error))
|
|
continue
|
|
logfd.write("\n{} (revision {})".format(title, page.revision_id))
|
|
logfd.flush()
|
|
|
|
process_text(page.content, lang)
|
|
try:
|
|
links = page.links
|
|
random.shuffle(links)
|
|
if len(links) > max_titles:
|
|
links = links[:max_titles]
|
|
next_titles += links
|
|
except KeyError:
|
|
pass
|
|
|
|
if depth >= options.max_depth:
|
|
return
|
|
|
|
random.shuffle(next_titles)
|
|
visit_pages (next_titles, depth + 1, lang, logfd)
|
|
|
|
language_c = lang.name.replace('-', '_').title()
|
|
build_log = current_dir + '/BuildLangModelLogs/Lang{}Model.log'.format(language_c)
|
|
logfd = open(build_log, 'w')
|
|
logfd.write('= Logs of language model for {} ({}) =\n'.format(lang.name, lang.code))
|
|
logfd.write('\n- Generated by {}'.format(os.path.basename(__file__)))
|
|
logfd.write('\n- Started: {}'.format(str(datetime.datetime.now())))
|
|
logfd.write('\n- Maximum depth: {}'.format(options.max_depth))
|
|
if options.max_page is not None:
|
|
logfd.write('\n- Max number of pages: {}'.format(options.max_page))
|
|
logfd.write('\n\n== Parsed pages ==\n')
|
|
logfd.flush()
|
|
try:
|
|
visit_pages(lang.start_pages, 0, lang, logfd)
|
|
except requests.exceptions.ConnectionError:
|
|
sys.stderr.write('Error: connection to Wikipedia failed. Aborting\n')
|
|
exit(1)
|
|
logfd.write('\n\n== End of Parsed pages ==')
|
|
logfd.write('\n\n- Wikipedia parsing ended at: {}\n'.format(str(datetime.datetime.now())))
|
|
logfd.flush()
|
|
|
|
########### CHARACTERS ###########
|
|
|
|
# Character ratios.
|
|
ratios = {}
|
|
n_char = len(characters)
|
|
occurrences = sum(characters.values())
|
|
|
|
logfd.write("\n{} characters appeared {} times.\n".format(n_char, occurrences))
|
|
for char in characters:
|
|
ratios[char] = characters[char] / occurrences
|
|
#logfd.write("Character '{}' usage: {} ({} %)\n".format(chr(char),
|
|
# characters[char],
|
|
# ratios[char] * 100))
|
|
|
|
sorted_ratios = sorted(ratios.items(), key=operator.itemgetter(1),
|
|
reverse=True)
|
|
# Accumulated ratios of the frequent chars.
|
|
accumulated_ratios = 0
|
|
|
|
# If there is no alphabet defined, we just use the first 64 letters, which was
|
|
# the original default.
|
|
# If there is an alphabet, we make sure all the alphabet characters are in the
|
|
# frequent list, and we stop then. There may therefore be more or less than
|
|
# 64 frequent characters depending on the language.
|
|
logfd.write('\nMost Frequent characters:')
|
|
very_freq_count = 0
|
|
very_freq_ratio = 0
|
|
if lang.alphabet is None and lang.frequent_ranges is None:
|
|
freq_count = min(64, len(sorted_ratios))
|
|
for order, (char, ratio) in enumerate(sorted_ratios):
|
|
if order >= freq_count:
|
|
break
|
|
logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
|
|
accumulated_ratios += ratio
|
|
if very_freq_ratio < 0.4:
|
|
very_freq_count += 1
|
|
very_freq_ratio += ratio
|
|
elif lang.alphabet is not None:
|
|
freq_count = 0
|
|
for order, (char, ratio) in enumerate(sorted_ratios):
|
|
if len(lang.alphabet) == 0:
|
|
break
|
|
if chr(char) in lang.alphabet:
|
|
lang.alphabet.remove(chr(char))
|
|
logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
|
|
accumulated_ratios += ratio
|
|
freq_count += 1
|
|
if very_freq_ratio < 0.4:
|
|
very_freq_count += 1
|
|
very_freq_ratio += ratio
|
|
else:
|
|
if len(lang.alphabet) > 0:
|
|
sys.stderr.write("Error: alphabet characters are absent from data collection"
|
|
"\n Please check the configuration or the data."
|
|
"\n Missing characters: {}".format(", ".join(lang.alphabet)))
|
|
exit(1)
|
|
elif lang.frequent_ranges is not None:
|
|
# How many characters in the frequent range?
|
|
frequent_ranges_size = 0
|
|
for start, end in lang.frequent_ranges:
|
|
frequent_ranges_size += end - start + 1
|
|
|
|
# Keep ratio for at least all the characters inside the frequent
|
|
# ranges.
|
|
freq_count = 0
|
|
for order, (char, ratio) in enumerate(sorted_ratios):
|
|
for start, end in lang.frequent_ranges:
|
|
if char >= start and char <= end:
|
|
freq_count += 1
|
|
accumulated_ratios += ratio
|
|
logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
|
|
frequent_ranges_size -= 1
|
|
break
|
|
else:
|
|
# A frequent character in the non-frequent range.
|
|
logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
|
|
freq_count += 1
|
|
accumulated_ratios += ratio
|
|
|
|
if very_freq_ratio < 0.4:
|
|
very_freq_count += 1
|
|
very_freq_ratio += ratio
|
|
|
|
if frequent_ranges_size <= 0:
|
|
break
|
|
|
|
low_freq_order = freq_count - 1
|
|
low_freq_ratio = 0
|
|
for back_order, (char, ratio) in enumerate(reversed(sorted_ratios[:freq_count])):
|
|
if low_freq_ratio < 0.03:
|
|
low_freq_ratio += ratio
|
|
low_freq_order -= 1
|
|
else:
|
|
break
|
|
|
|
logfd.write("\n\nThe first {} characters have an accumulated ratio of {}.\n".format(freq_count, accumulated_ratios))
|
|
logfd.write("The first {} characters have an accumulated ratio of {}.\n".format(very_freq_count, very_freq_ratio))
|
|
logfd.write("All characters whose order is over {} have an accumulated ratio of {}.\n".format(low_freq_order, low_freq_ratio))
|
|
|
|
with open(current_dir + '/header-template.cpp', 'r') as header_fd:
|
|
c_code = header_fd.read()
|
|
|
|
c_code += '\n#include "../nsSBCharSetProber.h"'
|
|
c_code += '\n#include "../nsSBCharSetProber-generated.h"'
|
|
c_code += '\n#include "../nsLanguageDetector.h"\n'
|
|
c_code += '\n#include "../nsLanguageDetector-generated.h"\n'
|
|
c_code += '\n/********* Language model for: {} *********/\n\n'.format(lang.name)
|
|
c_code += '/**\n * Generated by {}\n'.format(os.path.basename(__file__))
|
|
c_code += ' * On: {}\n'.format(str(datetime.datetime.now()))
|
|
c_code += ' **/\n'
|
|
|
|
c_code += \
|
|
"""
|
|
/* Character Mapping Table:
|
|
* ILL: illegal character.
|
|
* CTR: control character specific to the charset.
|
|
* RET: carriage/return.
|
|
* SYM: symbol (punctuation) that does not belong to word.
|
|
* NUM: 0 - 9.
|
|
*
|
|
* Other characters are ordered by probabilities
|
|
* (0 is the most common character in the language).
|
|
*
|
|
* Orders are generic to a language. So the codepoint with order X in
|
|
* CHARSET1 maps to the same character as the codepoint with the same
|
|
* order X in CHARSET2 for the same language.
|
|
* As such, it is possible to get missing order. For instance the
|
|
* ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1
|
|
* even though they are both used for French. Same for the euro sign.
|
|
*/
|
|
"""
|
|
|
|
for charset in lang_charsets:
|
|
charset_c = charset.replace('-', '_').title()
|
|
CTOM_str = 'static const unsigned char {}_CharToOrderMap[]'.format(charset_c)
|
|
CTOM_str += ' =\n{'
|
|
for line in range(0, 16):
|
|
CTOM_str += '\n '
|
|
for column in range(0, 16):
|
|
cp = line * 16 + column
|
|
cp_type = lang_charsets[charset].charmap[cp]
|
|
if cp_type == ILL:
|
|
CTOM_str += 'ILL,'
|
|
elif cp_type == RET:
|
|
CTOM_str += 'RET,'
|
|
elif cp_type == CTR:
|
|
CTOM_str += 'CTR,'
|
|
elif cp_type == SYM:
|
|
CTOM_str += 'SYM,'
|
|
elif cp_type == NUM:
|
|
CTOM_str += 'NUM,'
|
|
else: # LET
|
|
try:
|
|
uchar = bytes([cp]).decode(charset)
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Unknown character 0X{:X} in {}.'.format(cp, charset))
|
|
sys.stderr.write('Please verify your charset specification.\n')
|
|
exit(1)
|
|
except LookupError:
|
|
# Unknown encoding. Use iconv instead.
|
|
try:
|
|
call = subprocess.Popen(['iconv', '-t', 'UTF-8', '-f', charset],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
if call.poll() is not None:
|
|
(_, error) = call.communicate(input='')
|
|
sys.stderr.write('Error: `iconv` ended with error "{}".\n'.format(error))
|
|
exit(1)
|
|
(uchar, _) = call.communicate(input=bytes([cp]))
|
|
uchar = uchar.decode('UTF-8')
|
|
except FileNotFoundError:
|
|
sys.stderr.write('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
|
|
exit(1)
|
|
if len(uchar) == 0:
|
|
sys.stderr.write('TypeError: iconv failed to return a unicode character for codepoint "{}" in charset {}.\n'.format(hex(cp), charset))
|
|
exit(1)
|
|
#if lang.case_mapping and uchar.isupper() and \
|
|
#len(unicodedata.normalize('NFC', uchar.lower())) == 1:
|
|
# Unless we encounter special cases of characters with no
|
|
# composed lowercase, we lowercase it.
|
|
if lang.case_mapping or lang.custom_case_mapping is not None:
|
|
uchar = local_lowercase(uchar, lang)
|
|
if lang.alphabet_mapping is not None and uchar in lang.alphabet_mapping:
|
|
uchar = lang.alphabet_mapping[uchar]
|
|
for order, (char, ratio) in enumerate(sorted_ratios):
|
|
if char == ord(uchar):
|
|
CTOM_str += '{:3},'.format(min(249, order))
|
|
break
|
|
else:
|
|
# XXX: we must make sure the character order does not go
|
|
# over the special characters (250 currently). This may
|
|
# actually happen when building a model for a language
|
|
# writable with many different encoding. So let's just
|
|
# ceil the order value at 249 max.
|
|
# It may be an interesting alternative to add another
|
|
# constant for any character with an order > freqCharCount.
|
|
# Maybe IRR (irrelevant character) or simply CHR.
|
|
CTOM_str += '{:3},'.format(min(249, n_char))
|
|
n_char += 1
|
|
CTOM_str += ' /* {:X}X */'.format(line)
|
|
CTOM_str += '\n};\n/*'
|
|
CTOM_str += 'X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF'
|
|
CTOM_str += ' */\n\n'
|
|
c_code += CTOM_str
|
|
|
|
## UNICODE frequency.
|
|
|
|
# Since we can't map the full character table from encoding to order,
|
|
# just create a list from the most common characters from the language.
|
|
# The list is ordered by unicode code points (hence can be used
|
|
# generically for various encoding scheme as it is not encoding
|
|
# specific) allowing to search from code points efficiently by a divide
|
|
# and conqueer search algorithm.
|
|
# Each code point is immediately followed by its order.
|
|
|
|
# Keep the freq_count more frequent characters.
|
|
sorted_chars = [(char, freq, order) for order, (char, freq) in
|
|
enumerate(sorted_ratios)][:freq_count]
|
|
max_order = len(sorted_chars)
|
|
|
|
# Add equivalency characters.
|
|
equivalent = []
|
|
if lang.case_mapping:
|
|
for char, ratio, order in sorted_chars:
|
|
uppercased = chr(char).upper()
|
|
try:
|
|
if char != ord(uppercased):
|
|
equivalent += [(ord(uppercased), ratio, order)]
|
|
except TypeError:
|
|
# This happens for some case such as 'SS' as uppercase of 'ß'.
|
|
# Just ignore such cases.
|
|
sys.stderr.write("Ignoring '{}' as uppercase equivalent of '{}'.\n".format(uppercased, char))
|
|
|
|
if lang.alphabet_mapping is not None:
|
|
for alt_c in lang.alphabet_mapping:
|
|
for char, ratio, order in sorted_chars:
|
|
if alt_c == chr(char):
|
|
sys.stderr.write("ALREADY {}\n".format(alt_c))
|
|
exit(1)
|
|
elif char == ord(lang.alphabet_mapping[alt_c]):
|
|
equivalent += [(ord(alt_c), ratio, order)]
|
|
break
|
|
else:
|
|
sys.stderr.write("Base equivalent for {} not found in frequent characters!\n".format(alt_c))
|
|
exit(1)
|
|
|
|
sorted_chars += equivalent
|
|
|
|
# Order by code point.
|
|
sorted_chars = sorted(sorted_chars, key=operator.itemgetter(0))
|
|
|
|
CTOM_str = 'static const int Unicode_Char_size = {};\n'.format(len(sorted_chars))
|
|
|
|
CTOM_str += 'static const unsigned int Unicode_CharOrder[]'
|
|
CTOM_str += ' =\n{'
|
|
column = 0
|
|
|
|
max_char_width = math.floor(math.log10(sorted_chars[-1][0])) + 1
|
|
max_order_width = math.floor(math.log10(max_order)) + 1
|
|
|
|
for char, ratio, order in sorted_chars:
|
|
if column % 8 == 0:
|
|
CTOM_str += '\n '
|
|
column += 1
|
|
CTOM_str += '{}{:>{width}}, '.format('' if column % 8 == 0 else ' ', char, width=max_char_width)
|
|
CTOM_str += '{:>{width}},'.format(order, width=max_order_width)
|
|
|
|
CTOM_str += '\n};\n\n'
|
|
c_code += CTOM_str
|
|
|
|
########### SEQUENCES ###########
|
|
|
|
ratios = {}
|
|
occurrences = sum(sequences.values())
|
|
|
|
accumulated_seq_count = 0
|
|
order_3 = -1
|
|
order_2 = -1
|
|
ratio_3 = -1
|
|
ratio_2 = -1
|
|
count_512 = -1
|
|
count_1024 = -1
|
|
sorted_seqs = sorted(sequences.items(), key=operator.itemgetter(1),
|
|
reverse=True)
|
|
for order, ((c1, c2), count) in enumerate(sorted_seqs):
|
|
accumulated_seq_count += count
|
|
if order_3 == -1 and accumulated_seq_count / occurrences >= 0.995:
|
|
order_3 = order
|
|
ratio_3 = accumulated_seq_count / occurrences
|
|
elif order_2 == -1 and accumulated_seq_count / occurrences >= 0.999:
|
|
order_2 = order
|
|
ratio_2 = accumulated_seq_count / occurrences
|
|
if order < 512:
|
|
count_512 += count
|
|
elif order < 1024:
|
|
count_1024 += count
|
|
|
|
if order_3 != -1 and order_2 != -1:
|
|
break
|
|
|
|
if order_3 == -1 or order_2 == -1:
|
|
# This would probably never happens. It would require a language with
|
|
# very few possible sequences and each of the sequences are widely
|
|
# used. Just add this code for completio, but it won't likely ever be
|
|
# run.
|
|
order_2 = 512
|
|
order_3 = 1024
|
|
ratio_2 = count_512 / occurrences
|
|
ratio_3 = count_1024 / occurrences
|
|
|
|
logfd.write("\n{} sequences found.\n".format(len(sorted_seqs)))
|
|
|
|
c_code += """
|
|
/* Model Table:
|
|
* Total considered sequences: {} / {}
|
|
* - Positive sequences: first {} ({})
|
|
* - Probable sequences: next {} ({}-{}) ({})
|
|
* - Neutral sequences: last {} ({})
|
|
* - Negative sequences: {} (off-ratio)
|
|
* Negative sequences: TODO""".format(len(sorted_seqs),
|
|
freq_count * freq_count,
|
|
order_3, ratio_3,
|
|
order_2 - order_3,
|
|
order_2, order_3,
|
|
ratio_2 - ratio_3,
|
|
freq_count * freq_count - order_2,
|
|
1 - ratio_2,
|
|
freq_count * freq_count - len(sorted_seqs))
|
|
|
|
logfd.write("\nFirst {} (typical positive ratio): {}".format(order_3, ratio_3))
|
|
logfd.write("\nNext {} ({}-{}): {}".format(order_2 - order_3,
|
|
order_2, order_3,
|
|
ratio_2 - ratio_3))
|
|
logfd.write("\nRest: {}".format(1 - ratio_2))
|
|
|
|
c_code += "\n */\n"
|
|
|
|
LM_str = 'static const PRUint8 {}LangModel[]'.format(language_c)
|
|
LM_str += ' =\n{'
|
|
for line in range(0, freq_count):
|
|
LM_str += '\n '
|
|
for column in range(0, freq_count):
|
|
# Let's not make too long lines.
|
|
if freq_count > 40 and column == int(freq_count / 2):
|
|
LM_str += '\n '
|
|
first_order = int(line)
|
|
second_order = column
|
|
if first_order < len(sorted_ratios) and second_order < len(sorted_ratios):
|
|
(first_char, _) = sorted_ratios[first_order]
|
|
(second_char, _) = sorted_ratios[second_order]
|
|
if (first_char, second_char) in sequences:
|
|
for order, (seq, _) in enumerate(sorted_seqs):
|
|
if seq == (first_char, second_char):
|
|
if order < order_3:
|
|
LM_str += '3,'
|
|
elif order < order_2:
|
|
LM_str += '2,'
|
|
else:
|
|
LM_str += '1,'
|
|
break
|
|
else:
|
|
pass # impossible!
|
|
LM_str += '0,'
|
|
else:
|
|
LM_str += '0,'
|
|
else:
|
|
# It may indeed happen that we find less than 64 letters used for a
|
|
# given language.
|
|
LM_str += '0,'
|
|
LM_str += '\n};\n'
|
|
c_code += LM_str
|
|
|
|
for charset in lang_charsets:
|
|
charset_c = charset.replace('-', '_').title()
|
|
SM_str = '\n\nconst SequenceModel {}{}Model ='.format(charset_c, language_c)
|
|
SM_str += '\n{\n '
|
|
SM_str += '{}_CharToOrderMap,\n {}LangModel,'.format(charset_c, language_c)
|
|
SM_str += '\n {},'.format(freq_count)
|
|
SM_str += '\n (float){},'.format(ratio_2)
|
|
SM_str += '\n {},'.format('PR_TRUE' if lang.use_ascii else 'PR_FALSE')
|
|
SM_str += '\n "{}",'.format(charset)
|
|
SM_str += '\n "{}"'.format(lang.code)
|
|
SM_str += '\n};'
|
|
c_code += SM_str
|
|
|
|
SM_str = '\n\nconst LanguageModel {}Model ='.format(language_c)
|
|
SM_str += '\n{'
|
|
SM_str += '\n "{}",'.format(lang.code)
|
|
SM_str += '\n Unicode_CharOrder,'
|
|
SM_str += '\n {},'.format(len(sorted_chars)) # Order is wrong!
|
|
SM_str += '\n {}LangModel,'.format(language_c)
|
|
SM_str += '\n {},'.format(freq_count)
|
|
SM_str += '\n {},'.format(very_freq_count)
|
|
SM_str += '\n (float){},'.format(very_freq_ratio)
|
|
SM_str += '\n {},'.format(low_freq_order)
|
|
SM_str += '\n (float){},'.format(low_freq_ratio)
|
|
SM_str += '\n};'
|
|
c_code += SM_str
|
|
|
|
c_code += '\n'
|
|
|
|
lang_model_file = current_dir + '/../src/LangModels/Lang{}Model.cpp'.format(language_c)
|
|
with open(lang_model_file, 'w') as cpp_fd:
|
|
cpp_fd.write(c_code)
|
|
|
|
logfd.write('\n\n- Processing end: {}\n'.format(str(datetime.datetime.now())))
|
|
logfd.close()
|
|
|
|
generated_files += [ (lang_model_file, build_log) ]
|
|
|
|
charset_cpp = os.path.join(current_dir, '../src', 'nsSBCharSetProber-generated.h')
|
|
print("\nGenerating {}…".format(charset_cpp))
|
|
|
|
with open(charset_cpp, 'w') as cpp_fd:
|
|
with open(current_dir + '/header-template.cpp', 'r') as header_fd:
|
|
cpp_fd.write(header_fd.read())
|
|
|
|
cpp_fd.write('\n#ifndef nsSingleByteCharSetProber_generated_h__')
|
|
cpp_fd.write('\n#define nsSingleByteCharSetProber_generated_h__\n')
|
|
|
|
all_extern_declarations = ''
|
|
n_sequence_models = 0
|
|
for l in all_langs:
|
|
l = l.lower()
|
|
# Load the language data.
|
|
sys_path_backup = sys.path
|
|
sys.path = [current_dir + '/langs']
|
|
try:
|
|
lang = importlib.import_module(l)
|
|
except ImportError:
|
|
sys.stderr.write('Unknown language code "{}": '
|
|
'file "langs/{}.py" does not exist.'.format(l, l))
|
|
exit(1)
|
|
sys.path = sys_path_backup
|
|
|
|
language_c = lang.name.replace('-', '_').title()
|
|
lang_charsets = charsets.db.load(lang.charsets)
|
|
for charset in lang_charsets:
|
|
charset_c = charset.replace('-', '_').title()
|
|
all_extern_declarations += '\nextern const SequenceModel {}{}Model;'.format(charset_c, language_c)
|
|
n_sequence_models += 1
|
|
all_extern_declarations += '\n'
|
|
|
|
cpp_fd.write('\n#define NUM_OF_SEQUENCE_MODELS {}\n'.format(n_sequence_models))
|
|
cpp_fd.write('{}'.format(all_extern_declarations))
|
|
cpp_fd.write('\n#endif /* nsSingleByteCharSetProber_generated_h__ */')
|
|
|
|
print("Done!")
|
|
|
|
language_cpp = os.path.join(current_dir, '../src', 'nsLanguageDetector-generated.h')
|
|
print("\nGenerating {}…".format(language_cpp))
|
|
|
|
with open(language_cpp, 'w') as cpp_fd:
|
|
with open(current_dir + '/header-template.cpp', 'r') as header_fd:
|
|
cpp_fd.write(header_fd.read())
|
|
|
|
cpp_fd.write('\n#ifndef nsLanguageDetector_h_generated_h__')
|
|
cpp_fd.write('\n#define nsLanguageDetector_h_generated_h__\n')
|
|
|
|
all_extern_declarations = ''
|
|
n_language_models = 0
|
|
for l in all_langs:
|
|
l = l.lower()
|
|
# Load the language data.
|
|
sys_path_backup = sys.path
|
|
sys.path = [current_dir + '/langs']
|
|
try:
|
|
lang = importlib.import_module(l)
|
|
except ImportError:
|
|
sys.stderr.write('Unknown language code "{}": '
|
|
'file "langs/{}.py" does not exist.'.format(l, l))
|
|
exit(1)
|
|
sys.path = sys_path_backup
|
|
|
|
language_c = lang.name.replace('-', '_').title()
|
|
all_extern_declarations += '\nextern const LanguageModel {}Model;'.format(language_c)
|
|
n_language_models += 1
|
|
|
|
cpp_fd.write('\n#define NUM_OF_LANGUAGE_MODELS {}\n'.format(n_language_models))
|
|
cpp_fd.write('{}'.format(all_extern_declarations))
|
|
cpp_fd.write('\n\n#endif /* nsLanguageDetector_h_generated_h__ */')
|
|
|
|
print("Done!")
|
|
if len(generated_files) > 0:
|
|
print("\nThe following language files has been generated:")
|
|
for (lang_model_file, build_log) in generated_files:
|
|
print("\n- Language file: {}".format(lang_model_file))
|
|
print("\n Build log: {}".format(build_log))
|
|
|
|
print("\nTODO:")
|
|
print("- edit nsSBCSGroupProber::nsSBCSGroupProber() in src/nsSBCSGroupProber.cpp manually to test new sequence models;")
|
|
print("- edit nsMBCSGroupProber::nsMBCSGroupProber() in src/nsMBCSGroupProber.cpp manually to test new language models;")
|
|
print("- add any new language files to src/CMakeLists.txt;")
|
|
print("- commit generated files if tests are successful.")
|