archive-gh-me/jerryscript
1
0
Fork 0
mirror of https://github.com/jerryscript-project/jerryscript.git synced 2025-12-15 16:29:21 +00:00
Code Issues Projects Releases Wiki Activity

Refactor the generator scripts for unicode tables (#1623)

Browse Source 
Extract the source code generator methods into a separated `unicode_c_source.py` script.
Fix the generator scripts to make them compatible with both Python2 and Python3.
Remove pylint warnings.

JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com
This commit is contained in:
Robert Sipka 2017-03-16 09:32:18 +01:00 committed by László Langó
parent 818c9cd0b0
commit d77d4ae1c6
5 changed files with 654 additions and 746 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
jerry-core/lit/lit-unicode-conversions.inc.h
View File

@ -11,10 +11,11 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file is automatically generated by the unicode_case_conversion.py script. Do not edit!
*/
/* This file is automatically generated by the unicode_case_conversion.py script
* from UnicodeData-9.0.0.txt and SpecialCasing-9.0.0.txt files. Do not edit! */
/* Contains start points of character case ranges (these are bidirectional conversions). */
static const uint16_t jerry_character_case_ranges[] JERRY_CONST_DATA =
{
@ -154,9 +155,8 @@ static const uint16_t jerry_upper_case_conversions[] JERRY_CONST_DATA =
0x0046, 0x004c
};
/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
/* Number of one-to-one, one-to-two, and one-to-three uppercase conversions. */
static const uint8_t jerry_upper_case_conversion_counters[] JERRY_CONST_DATA =
{
0x001c, 0x002c, 0x0010
};

7
jerry-core/lit/lit-unicode-ranges.inc.h
View File

@ -11,11 +11,11 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file is automatically generated by the unicode_ranges.py script
* from UnicodeData-3.0.0.txt. Do not edit!
*/
/* This file is automatically generated by the unicode_ranges.py script
* from UnicodeData-3.0.0.txt. Do not edit! */
/**
* Character interval starting points for the unicode letters.
*
@ -180,4 +180,3 @@ static const uint16_t jerry_unicode_separator_chars[] JERRY_CONST_DATA =
{
0x1680, 0x180e, 0x202f, 0x205f, 0x3000
};

75
tools/unicode_c_source.py Normal file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env python
# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
LICENSE = """/* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/"""
class Source(object):
def __init__(self, filepath):
self.__filepath = filepath
self.__header = [LICENSE, ""]
self.__data = []
def complete_header(self, completion):
self.__header.append(completion)
self.__header.append("") # for an extra empty line
def add_table(self, table, table_name, table_type, table_descr):
self.__data.append(table_descr)
self.__data.append("static const %s jerry_%s[] JERRY_CONST_DATA =" % (table_type, table_name))
self.__data.append("{")
self.__data.append(format_code(table, 1))
self.__data.append("};")
self.__data.append("") # for an extra empty line
def generate(self):
with open(self.__filepath, 'w') as genereted_source:
genereted_source.write("\n".join(self.__header))
genereted_source.write("\n".join(self.__data))
def regroup(list_to_group, num):
return [list_to_group[i:i+num] for i in range(0, len(list_to_group), num)]
def hex_format(char):
if isinstance(char, str):
char = ord(char)
return "0x{:04x}".format(char)
def format_code(code, indent):
lines = []
# convert all characters to hex format
converted_code = [hex_format(char) for char in code]
# 10 hex number per line
for line in regroup(", ".join(converted_code), 10 * 8):
lines.append((' ' * indent) + line.strip())
return "\n".join(lines)

988
tools/unicode_case_conversion.py
View File

File diff suppressed because it is too large Load Diff

322
tools/unicode_ranges.py Normal file → Executable file
View File

@ -26,152 +26,146 @@
# connector punctuation: Pc
# separators: Zs
from __future__ import print_function
from settings import PROJECT_DIR
from unicode_c_source import Source
import argparse
import bisect
import csv
import itertools
import os
import sys
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_DIR = os.path.normpath(os.path.join(TOOLS_DIR, '..'))
C_SOURCE_FILE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
parser = argparse.ArgumentParser()
parser.add_argument('unicode_data',
metavar='FILE',
action='store',
help='specify the unicode data file')
parser.add_argument('--c-source',
metavar='FILE',
action='store',
default=C_SOURCE_FILE,
help='specify the output c source (default: %(default)s)')
script_args = parser.parse_args()
RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('unicode_data',
metavar='FILE',
action='store',
help='specify the unicode data file')
parser.add_argument('--c-source',
metavar='FILE',
action='store',
default=RANGES_C_SOURCE,
help='specify the output c source (default: %(default)s)')
script_args = parser.parse_args()
if not os.path.isfile(script_args.unicode_data) or not os.access(script_args.unicode_data, os.R_OK):
print('The %s file is missing or not readable!' % script_args.unicode_data)
sys.exit(1)
letters, non_letters, separators = read_categories()
letters, non_letters, separators = read_categories(script_args.unicode_data)
letters_list = list(ranges(letters))
letter_interval_sps, letter_interval_lengths, letter_chars = split_list(letters_list)
letter_tables = split_list(list(ranges(letters)))
non_letter_tables = split_list(list(ranges(non_letters)))
separator_tables = split_list(list(ranges(separators)))
non_letters_list = list(ranges(non_letters))
non_letter_interval_sps, non_letter_interval_lengths, non_letter_chars = split_list(non_letters_list)
c_source = Source(script_args.c_source)
separator_list = list(ranges(separators))
separator_interval_sps, separator_interval_lengths, separator_chars = split_list(separator_list)
header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
" * from %s. Do not edit! */" % os.path.basename(script_args.unicode_data),
""]
source = GenSource()
c_source.complete_header("\n".join(header_completion))
letter_interval_sps_desc = """/**
* Character interval starting points for the unicode letters.
*
* The characters covered by these intervals are from
* the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl
*/"""
source.add_table("uint16_t",
"unicode_letter_interval_sps",
letter_interval_sps,
letter_interval_sps_desc)
c_source.add_table(letter_tables[0],
"unicode_letter_interval_sps",
"uint16_t",
("/**\n"
" * Character interval starting points for the unicode letters.\n"
" *\n"
" * The characters covered by these intervals are from\n"
" * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl\n"
" */"))
letter_interval_lengths_desc = """/**
* Character lengths for the unicode letters.
*
* The characters covered by these intervals are from
* the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl
*/"""
source.add_table("uint8_t",
"unicode_letter_interval_lengths",
letter_interval_lengths,
letter_interval_lengths_desc)
c_source.add_table(letter_tables[1],
"unicode_letter_interval_lengths",
"uint8_t",
("/**\n"
" * Character lengths for the unicode letters.\n"
" *\n"
" * The characters covered by these intervals are from\n"
" * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl\n"
" */"))
letter_chars_desc = """/**
* Those unicode letter characters that are not inside any of
* the intervals specified in jerry_unicode_letter_interval_sps array.
*
* The characters are from the following Unicode categories:
* Lu, Ll, Lt, Lm, Lo, Nl
*/"""
source.add_table("uint16_t",
"unicode_letter_chars",
letter_chars,
letter_chars_desc)
c_source.add_table(letter_tables[2],
"unicode_letter_chars",
"uint16_t",
("/**\n"
" * Those unicode letter characters that are not inside any of\n"
" * the intervals specified in jerry_unicode_letter_interval_sps array.\n"
" *\n"
" * The characters are from the following Unicode categories:\n"
" * Lu, Ll, Lt, Lm, Lo, Nl\n"
" */"))
non_letter_interval_sps_desc = """/**
* Character interval starting points for non-letter character
* that can be used as a non-first character of an identifier.
*
* The characters covered by these intervals are from
* the following Unicode categories: Nd, Mn, Mc, Pc
*/"""
source.add_table("uint16_t",
"unicode_non_letter_ident_part_interval_sps",
non_letter_interval_sps,
non_letter_interval_sps_desc)
c_source.add_table(non_letter_tables[0],
"unicode_non_letter_ident_part_interval_sps",
"uint16_t",
("/**\n"
" * Character interval starting points for non-letter character\n"
" * that can be used as a non-first character of an identifier.\n"
" *\n"
" * The characters covered by these intervals are from\n"
" * the following Unicode categories: Nd, Mn, Mc, Pc\n"
" */"))
non_letter_interval_lengths_desc = """/**
* Character interval lengths for non-letter character
* that can be used as a non-first character of an identifier.
*
* The characters covered by these intervals are from
* the following Unicode categories: Nd, Mn, Mc, Pc
*/"""
source.add_table("uint8_t",
"unicode_non_letter_ident_part_interval_lengths",
non_letter_interval_lengths,
non_letter_interval_lengths_desc)
c_source.add_table(non_letter_tables[1],
"unicode_non_letter_ident_part_interval_lengths",
"uint8_t",
("/**\n"
" * Character interval lengths for non-letter character\n"
" * that can be used as a non-first character of an identifier.\n"
" *\n"
" * The characters covered by these intervals are from\n"
" * the following Unicode categories: Nd, Mn, Mc, Pc\n"
" */"))
non_letter_chars_desc = """/**
* Those non-letter characters that can be used as a non-first
* character of an identifier and not included in any of the intervals
* specified in jerry_unicode_non_letter_ident_part_interval_sps array.
*
* The characters are from the following Unicode categories:
* Nd, Mn, Mc, Pc
*/"""
source.add_table("uint16_t",
"unicode_non_letter_ident_part_chars",
non_letter_chars,
non_letter_chars_desc)
c_source.add_table(non_letter_tables[2],
"unicode_non_letter_ident_part_chars",
"uint16_t",
("/**\n"
" * Those non-letter characters that can be used as a non-first\n"
" * character of an identifier and not included in any of the intervals\n"
" * specified in jerry_unicode_non_letter_ident_part_interval_sps array.\n"
" *\n"
" * The characters are from the following Unicode categories:\n"
" * Nd, Mn, Mc, Pc\n"
" */"))
separator_interval_sps_desc = """/**
* Unicode separator character interval starting points from Unicode category: Zs
*/"""
source.add_table("uint16_t",
"unicode_separator_char_interval_sps",
separator_interval_sps,
separator_interval_sps_desc)
c_source.add_table(separator_tables[0],
"unicode_separator_char_interval_sps",
"uint16_t",
("/**\n"
" * Unicode separator character interval starting points from Unicode category: Zs\n"
" */"))
separator_interval_lengths_desc = """/**
* Unicode separator character interval lengths from Unicode category: Zs
*/"""
source.add_table("uint8_t",
"unicode_separator_char_interval_lengths",
separator_interval_lengths,
separator_interval_lengths_desc)
c_source.add_table(separator_tables[1],
"unicode_separator_char_interval_lengths",
"uint8_t",
("/**\n"
" * Unicode separator character interval lengths from Unicode category: Zs\n"
" */"))
separator_chars_desc = """/**
* Unicode separator characters that are not in the
* jerry_unicode_separator_char_intervals array.
*
* Unicode category: Zs
*/"""
source.add_table("uint16_t",
"unicode_separator_chars",
separator_chars,
separator_chars_desc)
c_source.add_table(separator_tables[2],
"unicode_separator_chars",
"uint16_t",
("/**\n"
" * Unicode separator characters that are not in the\n"
" * jerry_unicode_separator_char_intervals array.\n"
" *\n"
" * Unicode category: Zs\n"
" */"))
source.write_source()
c_source.generate()
def read_categories():
def read_categories(unicode_data_file):
"""
Read the corresponding unicode values and store them in category lists.
@ -186,7 +180,7 @@ def read_categories():
non_letters = []
separators = []
with open(script_args.unicode_data) as unicode_data:
with open(unicode_data_file) as unicode_data:
unicode_data_reader = csv.reader(unicode_data, delimiter=';')
for line in unicode_data_reader:
@ -228,10 +222,9 @@ def ranges(i):
:return: List of ranges.
"""
for a, b in itertools.groupby(enumerate(i), lambda (x, y): y - x):
b = list(b)
yield b[0][1], b[-1][1]
for _, group in itertools.groupby(enumerate(i), lambda q: (q[1] - q[0])):
group = list(group)
yield group[0][1], group[-1][1]
def split_list(category_list):
@ -241,87 +234,24 @@ def split_list(category_list):
:return: List of interval starting points, interval lengths and single chars
"""
unicode_category_interval_sps = []
unicode_category_interval_lengths = []
unicode_category_chars = []
interval_sps = []
interval_lengths = []
chars = []
for element in category_list:
interval_length = element[1] - element[0]
if interval_length == 0:
unicode_category_chars.append(element[0])
elif (interval_length > 255):
chars.append(element[0])
elif interval_length > 255:
for i in range(element[0], element[1], 256):
length = 255 if (element[1] - i > 255) else (element[1] - i)
unicode_category_interval_sps.append(i)
unicode_category_interval_lengths.append(length)
interval_sps.append(i)
interval_lengths.append(length)
else:
unicode_category_interval_sps.append(element[0])
unicode_category_interval_lengths.append(element[1] - element[0])
interval_sps.append(element[0])
interval_lengths.append(element[1] - element[0])
return unicode_category_interval_sps, unicode_category_interval_lengths, unicode_category_chars
class GenSource(object):
"""Class defines a default generated c source."""
def __init__(self):
self._data = []
header = """/* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file is automatically generated by the {SCRIPT} script
* from {UNICODES}. Do not edit!
*/
""".format(SCRIPT=os.path.basename(__file__), UNICODES=os.path.basename(script_args.unicode_data))
self._data.append(header)
def _regroup(self, l, n):
return [l[i:i+n] for i in range(0, len(l), n)]
def _hex_format(self, ch):
if isinstance(ch, str):
ch = ord(ch)
return "0x{:04x}".format(ch)
def _format_code(self, code, indent):
lines = []
# convert all characters to hex format
converted_code = map(self._hex_format, code)
# 10 hex number per line
for line in self._regroup(", ".join(converted_code), 10 * 8):
lines.append((' ' * indent) + line.strip())
return "\n".join(lines)
def add_table(self, type_name, array_name, table, description=""):
table_str = """{DESC}
static const {TYPE} jerry_{NAME}[] JERRY_CONST_DATA =
{{
{TABLE}
}};
""".format(DESC=description, TYPE=type_name, NAME=array_name, TABLE=self._format_code(table, 1))
self._data.append(table_str)
def write_source(self):
with open(script_args.c_source, 'w') as genereted_source:
genereted_source.write(''.join(self._data))
return interval_sps, interval_lengths, chars
if __name__ == "__main__":