mirror of
https://github.com/jerryscript-project/jerryscript.git
synced 2025-12-15 16:29:21 +00:00
Improve toLowerCase and toUpperCase functions. (#1575)
Language-sensitive mappings are not processed now. Fixes #323 JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com
This commit is contained in:
Robert Sipka
László Langó
parent
025a99ccbb
commit
1b5f839db9
@ -17,6 +17,10 @@
|
||||
#include "lit/lit-unicode-ranges.inc.h"
|
||||
#include "lit-strings.h"
|
||||
|
||||
#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
|
||||
#include "lit-unicode-conversions.inc.h"
|
||||
#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
|
||||
|
||||
#define NUM_OF_ELEMENTS(array) (sizeof (array) / sizeof ((array)[0]))
|
||||
|
||||
/**
|
||||
@ -458,6 +462,184 @@ lit_char_is_word_char (ecma_char_t c) /**< code unit */
|
||||
|| c == LIT_CHAR_UNDERSCORE);
|
||||
} /* lit_char_is_word_char */
|
||||
|
||||
#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
|
||||
|
||||
/**
|
||||
* Check if the specified character is in one of those tables which contain bidirectional conversions.
|
||||
*
|
||||
* @return the mapped character sequence of an ecma character, if it's in the table.
|
||||
* 0 - otherwise.
|
||||
*/
|
||||
static ecma_length_t
|
||||
search_in_bidirectional_conversion_tables (ecma_char_t character, /**< code unit */
|
||||
ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
|
||||
bool is_lowercase) /**< is lowercase conversion */
|
||||
{
|
||||
/* 1, Check if the specified character is part of the jerry_character_case_ranges table. */
|
||||
int number_of_case_ranges = NUM_OF_ELEMENTS (jerry_character_case_ranges);
|
||||
int conv_counter = 0;
|
||||
|
||||
for (int i = 0; i < number_of_case_ranges; i++)
|
||||
{
|
||||
if (i % 2 == 0 && i > 0)
|
||||
{
|
||||
conv_counter++;
|
||||
}
|
||||
|
||||
int range_length = jerry_character_case_range_lengths[conv_counter];
|
||||
ecma_char_t start_point = jerry_character_case_ranges[i];
|
||||
|
||||
if (start_point > character || character >= start_point + range_length)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int char_dist = character - start_point;
|
||||
|
||||
if (i % 2 == 0)
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? (ecma_char_t) (jerry_character_case_ranges[i + 1] + char_dist) : character;
|
||||
}
|
||||
else
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (jerry_character_case_ranges[i - 1] + char_dist);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* 2, Check if the specified character is part of the character_pair_ranges table. */
|
||||
int bottom = 0;
|
||||
int top = NUM_OF_ELEMENTS (jerry_character_pair_ranges) - 1;
|
||||
|
||||
while (bottom <= top)
|
||||
{
|
||||
int middle = (bottom + top) / 2;
|
||||
ecma_char_t current_sp = jerry_character_pair_ranges[middle];
|
||||
|
||||
if (current_sp <= character && character < current_sp + jerry_character_pair_range_lengths[middle])
|
||||
{
|
||||
int char_dist = character - current_sp;
|
||||
|
||||
if ((character - current_sp) % 2 == 0)
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? (ecma_char_t) (current_sp + char_dist + 1) : character;
|
||||
}
|
||||
else
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (current_sp + char_dist - 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (character > current_sp)
|
||||
{
|
||||
bottom = middle + 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
top = middle - 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* 3, Check if the specified character is part of the character_pairs table. */
|
||||
int number_of_character_pairs = NUM_OF_ELEMENTS (jerry_character_pairs);
|
||||
|
||||
for (int i = 0; i < number_of_character_pairs; i++)
|
||||
{
|
||||
if (character != jerry_character_pairs[i])
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i % 2 == 0)
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? jerry_character_pairs[i + 1] : character;
|
||||
}
|
||||
else
|
||||
{
|
||||
output_buffer_p[0] = is_lowercase ? character : jerry_character_pairs[i - 1];
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
} /* search_in_bidirectional_conversion_tables */
|
||||
|
||||
/**
|
||||
* Check if the specified character is in the given conversion table.
|
||||
*
|
||||
* @return the mapped character sequence of an ecma character, if it's in the table.
|
||||
* 0 - otherwise.
|
||||
*/
|
||||
static ecma_length_t
|
||||
search_in_conversion_table (ecma_char_t character, /**< code unit */
|
||||
ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
|
||||
const ecma_char_t *array, /**< array */
|
||||
const uint8_t *counters) /**< case_values counter */
|
||||
{
|
||||
int end_point = 0;
|
||||
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
int start_point = end_point;
|
||||
int size_of_case_value = i + 1;
|
||||
end_point += counters[i] * (size_of_case_value + 1);
|
||||
|
||||
int bottom = start_point;
|
||||
int top = end_point - size_of_case_value;
|
||||
|
||||
while (bottom <= top)
|
||||
{
|
||||
int middle = (bottom + top) / 2;
|
||||
|
||||
middle -= ((middle - bottom) % (size_of_case_value + 1));
|
||||
|
||||
ecma_char_t current = array[middle];
|
||||
|
||||
if (current == character)
|
||||
{
|
||||
ecma_length_t char_sequence = 1;
|
||||
|
||||
switch (size_of_case_value)
|
||||
{
|
||||
case 3:
|
||||
{
|
||||
output_buffer_p[2] = array[middle + 3];
|
||||
char_sequence++;
|
||||
/* FALLTHRU */
|
||||
}
|
||||
case 2:
|
||||
{
|
||||
output_buffer_p[1] = array[middle + 2];
|
||||
char_sequence++;
|
||||
/* FALLTHRU */
|
||||
}
|
||||
default:
|
||||
{
|
||||
output_buffer_p[0] = array[middle + 1];
|
||||
return char_sequence;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (character < current)
|
||||
{
|
||||
top = middle - (size_of_case_value + 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
bottom = middle + (size_of_case_value + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
} /* search_in_conversion_table */
|
||||
#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
|
||||
|
||||
/**
|
||||
* Returns the lowercase character sequence of an ecma character.
|
||||
*
|
||||
@ -471,8 +653,6 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */
|
||||
ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
|
||||
ecma_length_t buffer_size) /**< buffer size */
|
||||
{
|
||||
/* TODO: Needs a proper lower case implementation. See issue #323. */
|
||||
|
||||
JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH);
|
||||
|
||||
if (character >= LIT_CHAR_UPPERCASE_A && character <= LIT_CHAR_UPPERCASE_Z)
|
||||
@ -481,6 +661,41 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
|
||||
|
||||
ecma_length_t lowercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, true);
|
||||
|
||||
if (lowercase_sequence != 0)
|
||||
{
|
||||
return lowercase_sequence;
|
||||
}
|
||||
|
||||
int num_of_lowercase_ranges = NUM_OF_ELEMENTS (jerry_lower_case_ranges);
|
||||
|
||||
for (int i = 0, j = 0; i < num_of_lowercase_ranges; i += 2, j++)
|
||||
{
|
||||
int range_length = jerry_lower_case_range_lengths[j] - 1;
|
||||
ecma_char_t start_point = jerry_lower_case_ranges[i];
|
||||
|
||||
if (start_point <= character && character <= start_point + range_length)
|
||||
{
|
||||
output_buffer_p[0] = (ecma_char_t) (jerry_lower_case_ranges[i + 1] + (character - start_point));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
lowercase_sequence = search_in_conversion_table (character,
|
||||
output_buffer_p,
|
||||
jerry_lower_case_conversions,
|
||||
jerry_lower_case_conversion_counters);
|
||||
|
||||
if (lowercase_sequence != 0)
|
||||
{
|
||||
return lowercase_sequence;
|
||||
}
|
||||
|
||||
#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
|
||||
|
||||
output_buffer_p[0] = character;
|
||||
return 1;
|
||||
} /* lit_char_to_lower_case */
|
||||
@ -498,8 +713,6 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */
|
||||
ecma_char_t *output_buffer_p, /**< buffer for the result characters */
|
||||
ecma_length_t buffer_size) /**< buffer size */
|
||||
{
|
||||
/* TODO: Needs a proper upper case implementation. See issue #323. */
|
||||
|
||||
JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH);
|
||||
|
||||
if (character >= LIT_CHAR_LOWERCASE_A && character <= LIT_CHAR_LOWERCASE_Z)
|
||||
@ -508,6 +721,42 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
|
||||
|
||||
ecma_length_t uppercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, false);
|
||||
|
||||
if (uppercase_sequence != 0)
|
||||
{
|
||||
return uppercase_sequence;
|
||||
}
|
||||
|
||||
int num_of_upper_case_special_ranges = NUM_OF_ELEMENTS (jerry_upper_case_special_ranges);
|
||||
|
||||
for (int i = 0, j = 0; i < num_of_upper_case_special_ranges; i += 3, j++)
|
||||
{
|
||||
int range_length = jerry_upper_case_special_range_lengths[j];
|
||||
ecma_char_t start_point = jerry_upper_case_special_ranges[i];
|
||||
|
||||
if (start_point <= character && character <= start_point + range_length)
|
||||
{
|
||||
output_buffer_p[0] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 1] + (character - start_point));
|
||||
output_buffer_p[1] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 2]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
uppercase_sequence = search_in_conversion_table (character,
|
||||
output_buffer_p,
|
||||
jerry_upper_case_conversions,
|
||||
jerry_upper_case_conversion_counters);
|
||||
|
||||
if (uppercase_sequence != 0)
|
||||
{
|
||||
return uppercase_sequence;
|
||||
}
|
||||
|
||||
#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
|
||||
|
||||
output_buffer_p[0] = character;
|
||||
return 1;
|
||||
} /* lit_char_to_upper_case */
|
||||
|
||||
162
jerry-core/lit/lit-unicode-conversions.inc.h
Normal file
162
jerry-core/lit/lit-unicode-conversions.inc.h
Normal file
@ -0,0 +1,162 @@
|
||||
/* Copyright JS Foundation and other contributors, http://js.foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
* This file is automatically generated by the unicode_case_conversion.py script. Do not edit!
|
||||
*/
|
||||
|
||||
/* Contains start points of character case ranges (these are bidirectional conversions). */
|
||||
static const uint16_t jerry_character_case_ranges[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x00c0, 0x00e0, 0x00d8, 0x00f8, 0x0189, 0x0256, 0x01b1, 0x028a, 0x0388, 0x03ad,
|
||||
0x038e, 0x03cd, 0x0391, 0x03b1, 0x03a3, 0x03c3, 0x03fd, 0x037b, 0x0400, 0x0450,
|
||||
0x0410, 0x0430, 0x0531, 0x0561, 0x10a0, 0x2d00, 0x13a0, 0xab70, 0x13f0, 0x13f8,
|
||||
0x1f08, 0x1f00, 0x1f18, 0x1f10, 0x1f28, 0x1f20, 0x1f38, 0x1f30, 0x1f48, 0x1f40,
|
||||
0x1f68, 0x1f60, 0x1fb8, 0x1fb0, 0x1fba, 0x1f70, 0x1fc8, 0x1f72, 0x1fd8, 0x1fd0,
|
||||
0x1fda, 0x1f76, 0x1fe8, 0x1fe0, 0x1fea, 0x1f7a, 0x1ff8, 0x1f78, 0x1ffa, 0x1f7c,
|
||||
0x2160, 0x2170, 0x24b6, 0x24d0, 0x2c00, 0x2c30, 0x2c7e, 0x023f, 0xff21, 0xff41
|
||||
};
|
||||
|
||||
/* Interval lengths of start points in `character_case_ranges` table. */
|
||||
static const uint8_t jerry_character_case_range_lengths[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0017, 0x0007, 0x0002, 0x0002, 0x0003, 0x0002, 0x0011, 0x0009, 0x0003, 0x0010,
|
||||
0x0020, 0x0026, 0x0026, 0x0050, 0x0006, 0x0008, 0x0006, 0x0008, 0x0008, 0x0006,
|
||||
0x0008, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002,
|
||||
0x0010, 0x001a, 0x002f, 0x0002, 0x001a
|
||||
};
|
||||
|
||||
/* Contains the start points of bidirectional conversion ranges. */
|
||||
static const uint16_t jerry_character_pair_ranges[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0100, 0x0132, 0x0139, 0x014a, 0x0179, 0x0182, 0x0187, 0x018b, 0x0191, 0x0198,
|
||||
0x01a0, 0x01a7, 0x01ac, 0x01af, 0x01b3, 0x01b8, 0x01bc, 0x01cd, 0x01de, 0x01f4,
|
||||
0x01f8, 0x0222, 0x023b, 0x0241, 0x0246, 0x0370, 0x0376, 0x03d8, 0x03f7, 0x03fa,
|
||||
0x0460, 0x048a, 0x04c1, 0x04d0, 0x1e00, 0x1ea0, 0x2183, 0x2c60, 0x2c67, 0x2c72,
|
||||
0x2c75, 0x2c80, 0x2ceb, 0x2cf2, 0xa640, 0xa680, 0xa722, 0xa732, 0xa779, 0xa77e,
|
||||
0xa78b, 0xa790, 0xa796, 0xa7b4
|
||||
};
|
||||
|
||||
/* Interval lengths of start points in `character_pair_ranges` table. */
|
||||
static const uint8_t jerry_character_pair_range_lengths[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0030, 0x0006, 0x0010, 0x002e, 0x0006, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002,
|
||||
0x0006, 0x0002, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0010, 0x0012, 0x0002,
|
||||
0x0028, 0x0012, 0x0002, 0x0002, 0x000a, 0x0004, 0x0002, 0x0018, 0x0002, 0x0002,
|
||||
0x0022, 0x0036, 0x000e, 0x0060, 0x0096, 0x0060, 0x0002, 0x0002, 0x0006, 0x0002,
|
||||
0x0002, 0x0064, 0x0004, 0x0002, 0x002e, 0x001c, 0x000e, 0x003e, 0x0004, 0x000a,
|
||||
0x0002, 0x0004, 0x0014, 0x0004
|
||||
};
|
||||
|
||||
/* Contains lower/upper case bidirectional conversion pairs. */
|
||||
static const uint16_t jerry_character_pairs[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0178, 0x00ff, 0x0181, 0x0253, 0x0186, 0x0254, 0x018e, 0x01dd, 0x018f, 0x0259,
|
||||
0x0190, 0x025b, 0x0193, 0x0260, 0x0194, 0x0263, 0x0196, 0x0269, 0x0197, 0x0268,
|
||||
0x019c, 0x026f, 0x019d, 0x0272, 0x019f, 0x0275, 0x01a6, 0x0280, 0x01a9, 0x0283,
|
||||
0x01ae, 0x0288, 0x01b7, 0x0292, 0x01c4, 0x01c6, 0x01c7, 0x01c9, 0x01ca, 0x01cc,
|
||||
0x01f1, 0x01f3, 0x01f6, 0x0195, 0x01f7, 0x01bf, 0x0220, 0x019e, 0x023a, 0x2c65,
|
||||
0x023d, 0x019a, 0x023e, 0x2c66, 0x0243, 0x0180, 0x0244, 0x0289, 0x0245, 0x028c,
|
||||
0x037f, 0x03f3, 0x0386, 0x03ac, 0x038c, 0x03cc, 0x03cf, 0x03d7, 0x03f9, 0x03f2,
|
||||
0x04c0, 0x04cf, 0x10c7, 0x2d27, 0x10cd, 0x2d2d, 0x1f59, 0x1f51, 0x1f5b, 0x1f53,
|
||||
0x1f5d, 0x1f55, 0x1f5f, 0x1f57, 0x1fec, 0x1fe5, 0x2132, 0x214e, 0x2c62, 0x026b,
|
||||
0x2c63, 0x1d7d, 0x2c64, 0x027d, 0x2c6d, 0x0251, 0x2c6e, 0x0271, 0x2c6f, 0x0250,
|
||||
0x2c70, 0x0252, 0xa77d, 0x1d79, 0xa78d, 0x0265, 0xa7aa, 0x0266, 0xa7ab, 0x025c,
|
||||
0xa7ac, 0x0261, 0xa7ad, 0x026c, 0xa7ae, 0x026a, 0xa7b0, 0x029e, 0xa7b1, 0x0287,
|
||||
0xa7b2, 0x029d, 0xa7b3, 0xab53
|
||||
};
|
||||
|
||||
/* Contains start points of one-to-two uppercase ranges where the second character
|
||||
* is always the same.
|
||||
*/
|
||||
static const uint16_t jerry_upper_case_special_ranges[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x1f80, 0x1f08, 0x0399, 0x1f88, 0x1f08, 0x0399, 0x1f90, 0x1f28, 0x0399, 0x1f98,
|
||||
0x1f28, 0x0399, 0x1fa0, 0x1f68, 0x0399, 0x1fa8, 0x1f68, 0x0399
|
||||
};
|
||||
|
||||
/* Interval lengths for start points in `upper_case_special_ranges` table. */
|
||||
static const uint8_t jerry_upper_case_special_range_lengths[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0007, 0x0007, 0x0007, 0x0007, 0x0007, 0x0007
|
||||
};
|
||||
|
||||
/* Contains start points of lowercase ranges. */
|
||||
static const uint16_t jerry_lower_case_ranges[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x1e96, 0x1e96, 0x1f80, 0x1f80, 0x1f88, 0x1f80, 0x1f90, 0x1f90, 0x1f98, 0x1f90,
|
||||
0x1fa0, 0x1fa0, 0x1fa8, 0x1fa0, 0x1fb2, 0x1fb2, 0x1fb6, 0x1fb6, 0x1fc2, 0x1fc2,
|
||||
0x1fc6, 0x1fc6, 0x1fd2, 0x1fd2, 0x1fd6, 0x1fd6, 0x1fe2, 0x1fe2, 0x1fe6, 0x1fe6,
|
||||
0x1ff2, 0x1ff2, 0x1ff6, 0x1ff6, 0xfb00, 0xfb00, 0xfb13, 0xfb13
|
||||
};
|
||||
|
||||
/* Interval lengths for start points in `lower_case_ranges` table. */
|
||||
static const uint8_t jerry_lower_case_range_lengths[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0005, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0003, 0x0002, 0x0003,
|
||||
0x0002, 0x0002, 0x0002, 0x0003, 0x0002, 0x0003, 0x0002, 0x0007, 0x0005
|
||||
};
|
||||
|
||||
/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */
|
||||
static const uint16_t jerry_lower_case_conversions[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x00df, 0x00df, 0x0149, 0x0149, 0x01c5, 0x01c6, 0x01c8, 0x01c9, 0x01cb, 0x01cc,
|
||||
0x01f0, 0x01f0, 0x01f2, 0x01f3, 0x0390, 0x0390, 0x03b0, 0x03b0, 0x03f4, 0x03b8,
|
||||
0x0587, 0x0587, 0x1e9e, 0x00df, 0x1f50, 0x1f50, 0x1f52, 0x1f52, 0x1f54, 0x1f54,
|
||||
0x1f56, 0x1f56, 0x1fbc, 0x1fb3, 0x1fcc, 0x1fc3, 0x1ffc, 0x1ff3, 0x2126, 0x03c9,
|
||||
0x212a, 0x006b, 0x212b, 0x00e5, 0x0130, 0x0069, 0x0307
|
||||
};
|
||||
|
||||
/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
|
||||
static const uint8_t jerry_lower_case_conversion_counters[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x0016, 0x0001, 0x0000
|
||||
};
|
||||
|
||||
/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */
|
||||
static const uint16_t jerry_upper_case_conversions[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x00b5, 0x039c, 0x0130, 0x0130, 0x0131, 0x0049, 0x017f, 0x0053, 0x01c5, 0x01c4,
|
||||
0x01c8, 0x01c7, 0x01cb, 0x01ca, 0x01f2, 0x01f1, 0x0345, 0x0399, 0x03c2, 0x03a3,
|
||||
0x03d0, 0x0392, 0x03d1, 0x0398, 0x03d5, 0x03a6, 0x03d6, 0x03a0, 0x03f0, 0x039a,
|
||||
0x03f1, 0x03a1, 0x03f5, 0x0395, 0x1c80, 0x0412, 0x1c81, 0x0414, 0x1c82, 0x041e,
|
||||
0x1c83, 0x0421, 0x1c84, 0x0422, 0x1c85, 0x0422, 0x1c86, 0x042a, 0x1c87, 0x0462,
|
||||
0x1c88, 0xa64a, 0x1e9b, 0x1e60, 0x1fbe, 0x0399, 0x00df, 0x0053, 0x0053, 0x0149,
|
||||
0x02bc, 0x004e, 0x01f0, 0x004a, 0x030c, 0x0587, 0x0535, 0x0552, 0x1e96, 0x0048,
|
||||
0x0331, 0x1e97, 0x0054, 0x0308, 0x1e98, 0x0057, 0x030a, 0x1e99, 0x0059, 0x030a,
|
||||
0x1e9a, 0x0041, 0x02be, 0x1f50, 0x03a5, 0x0313, 0x1f87, 0x1f0f, 0x0399, 0x1f8f,
|
||||
0x1f0f, 0x0399, 0x1f97, 0x1f2f, 0x0399, 0x1f9f, 0x1f2f, 0x0399, 0x1fa7, 0x1f6f,
|
||||
0x0399, 0x1faf, 0x1f6f, 0x0399, 0x1fb2, 0x1fba, 0x0399, 0x1fb3, 0x0391, 0x0399,
|
||||
0x1fb4, 0x0386, 0x0399, 0x1fb6, 0x0391, 0x0342, 0x1fbc, 0x0391, 0x0399, 0x1fc2,
|
||||
0x1fca, 0x0399, 0x1fc3, 0x0397, 0x0399, 0x1fc4, 0x0389, 0x0399, 0x1fc6, 0x0397,
|
||||
0x0342, 0x1fcc, 0x0397, 0x0399, 0x1fd6, 0x0399, 0x0342, 0x1fe4, 0x03a1, 0x0313,
|
||||
0x1fe6, 0x03a5, 0x0342, 0x1ff2, 0x1ffa, 0x0399, 0x1ff3, 0x03a9, 0x0399, 0x1ff4,
|
||||
0x038f, 0x0399, 0x1ff6, 0x03a9, 0x0342, 0x1ffc, 0x03a9, 0x0399, 0xfb00, 0x0046,
|
||||
0x0046, 0xfb01, 0x0046, 0x0049, 0xfb02, 0x0046, 0x004c, 0xfb05, 0x0053, 0x0054,
|
||||
0xfb06, 0x0053, 0x0054, 0xfb13, 0x0544, 0x0546, 0xfb14, 0x0544, 0x0535, 0xfb15,
|
||||
0x0544, 0x053b, 0xfb16, 0x054e, 0x0546, 0xfb17, 0x0544, 0x053d, 0x0390, 0x0399,
|
||||
0x0308, 0x0301, 0x03b0, 0x03a5, 0x0308, 0x0301, 0x1f52, 0x03a5, 0x0313, 0x0300,
|
||||
0x1f54, 0x03a5, 0x0313, 0x0301, 0x1f56, 0x03a5, 0x0313, 0x0342, 0x1fb7, 0x0391,
|
||||
0x0342, 0x0399, 0x1fc7, 0x0397, 0x0342, 0x0399, 0x1fd2, 0x0399, 0x0308, 0x0300,
|
||||
0x1fd3, 0x0399, 0x0308, 0x0301, 0x1fd7, 0x0399, 0x0308, 0x0342, 0x1fe2, 0x03a5,
|
||||
0x0308, 0x0300, 0x1fe3, 0x03a5, 0x0308, 0x0301, 0x1fe7, 0x03a5, 0x0308, 0x0342,
|
||||
0x1ff7, 0x03a9, 0x0342, 0x0399, 0xfb03, 0x0046, 0x0046, 0x0049, 0xfb04, 0x0046,
|
||||
0x0046, 0x004c
|
||||
};
|
||||
|
||||
/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
|
||||
static const uint8_t jerry_upper_case_conversion_counters[] JERRY_CONST_DATA =
|
||||
{
|
||||
0x001c, 0x002c, 0x0010
|
||||
};
|
||||
|
||||
@ -10,4 +10,5 @@ CONFIG_DISABLE_NUMBER_BUILTIN
|
||||
CONFIG_DISABLE_REGEXP_BUILTIN
|
||||
CONFIG_DISABLE_STRING_BUILTIN
|
||||
CONFIG_DISABLE_TYPEDARRAY_BUILTIN
|
||||
CONFIG_DISABLE_UNICODE_CASE_CONVERSION
|
||||
|
||||
|
||||
@ -12,7 +12,72 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Conversion
|
||||
// LATIN SMALL LIGATURES
|
||||
// LATIN SMALL LIGATURE FF
|
||||
assert ("\ufb00".toLowerCase() == "\ufb00");
|
||||
assert ("\ufb00".toUpperCase() == "\u0046\u0046");
|
||||
// LATIN SMALL LIGATURE FI
|
||||
assert ("\ufb01".toLowerCase() == "\ufb01");
|
||||
assert ("\ufb01".toUpperCase() == "\u0046\u0049");
|
||||
// LATIN SMALL LIGATURE FL
|
||||
assert ("\ufb02".toLowerCase() == "\ufb02");
|
||||
assert ("\ufb02".toUpperCase() == "\u0046\u004c");
|
||||
// LATIN SMALL LIGATURE FFI
|
||||
assert ("\ufb03".toLowerCase() == "\ufb03");
|
||||
assert ("\ufb03".toUpperCase() == "\u0046\u0046\u0049");
|
||||
// LATIN SMALL LIGATURE FFL
|
||||
assert ("\ufb04".toLowerCase() == "\ufb04");
|
||||
assert ("\ufb04".toUpperCase() == "\u0046\u0046\u004c");
|
||||
// LATIN SMALL LIGATURE LONG S T
|
||||
assert ("\ufb05".toLowerCase() == "\ufb05");
|
||||
assert ("\ufb05".toUpperCase() == "\u0053\u0054");
|
||||
// LATIN SMALL LIGATURE ST
|
||||
assert ("\ufb06".toLowerCase() == "\ufb06");
|
||||
assert ("\ufb06".toUpperCase() == "\u0053\u0054");
|
||||
|
||||
// LATIN CAPITAL LETTER I WITH DOT ABOVE
|
||||
assert ("\u0130".toLowerCase() == "\u0069\u0307");
|
||||
assert ("\u0130".toUpperCase() == "\u0130");
|
||||
|
||||
// LATIN SMALL LETTER SHARP S
|
||||
assert ("\u00df".toLowerCase() == "\u00df");
|
||||
assert ("\u00df".toUpperCase() == "\u0053\u0053");
|
||||
|
||||
// LATIN CAPITAL LETTER I WITH BREVE
|
||||
assert ("\u012c".toLowerCase() == "\u012d");
|
||||
assert ("\u012c".toUpperCase() == "\u012c");
|
||||
// LATIN SMALL LETTER I WITH BREVE
|
||||
assert ("\u012d".toLowerCase() == "\u012d")
|
||||
assert ("\u012d".toUpperCase() == "\u012c");
|
||||
|
||||
// Check randomly selected characters from conversion tables
|
||||
|
||||
// lower-case conversions
|
||||
assert ("\u01c5\u01c8\u01cb\u212b".toLowerCase() == "\u01c6\u01c9\u01cc\u00e5");
|
||||
assert ("\u0130".toLowerCase() == "\u0069\u0307");
|
||||
|
||||
// upper-case conversions
|
||||
assert ("\u00b5\u017f".toUpperCase() == "\u039c\u0053");
|
||||
assert ("\ufb17\u00df\u1fbc".toUpperCase() == "\u0544\u053D\u0053\u0053\u0391\u0399");
|
||||
assert ("\ufb03\ufb04".toUpperCase() == "\u0046\u0046\u0049\u0046\u0046\u004c");
|
||||
|
||||
// character case ranges
|
||||
assert ("\u0100\u0101\u0139\u03fa\ua7b4".toLowerCase() == "\u0101\u0101\u013a\u03fb\ua7b5");
|
||||
assert ("\u0101\u0100\u013a\u03fb\ua7b5".toUpperCase() == "\u0100\u0100\u0139\u03fa\ua7b4");
|
||||
|
||||
// character pairs
|
||||
assert ("\u0178\ua7b1\u0287\ua7b3".toLowerCase() == "\u00ff\u0287\u0287\uab53");
|
||||
assert ("\u00ff\u0287\ua7b1\uab53".toUpperCase() == "\u0178\ua7b1\ua7b1\ua7b3");
|
||||
|
||||
// character case ranges
|
||||
assert ("\u00e0\u00c0\u00c1\u00c2\uff21".toLowerCase() == "\u00e0\u00e0\u00e1\u00e2\uff41");
|
||||
assert ("\u00e0\u00c0\u00e1\u00e2\uff41".toUpperCase() == "\u00c0\u00c0\u00c1\u00c2\uff21");
|
||||
|
||||
// lower-case ranges
|
||||
assert ("\u1f88\u1f98\u1fa8\u1f8b\u1faf".toLowerCase() == "\u1f80\u1f90\u1fa0\u1f83\u1fa7");
|
||||
|
||||
// upper-case special ranges
|
||||
assert ("\u1f80\u1f81\u1fa7".toUpperCase() == "\u1f08\u0399\u1f09\u0399\u1f6f\u0399");
|
||||
|
||||
assert ("0123456789abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXYZ".toLowerCase()
|
||||
== "0123456789abcdefghijklmnopqrstuvwxzyabcdefghijklmnopqrstuvwxyz");
|
||||
|
||||
681
tools/unicode_case_conversion.py
Executable file
681
tools/unicode_case_conversion.py
Executable file
@ -0,0 +1,681 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright JS Foundation and other contributors, http://js.foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import itertools
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
try:
|
||||
unichr
|
||||
except NameError:
|
||||
unichr = chr
|
||||
|
||||
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_DIR = os.path.normpath(os.path.join(TOOLS_DIR, '..'))
|
||||
C_SOURCE_FILE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
|
||||
|
||||
|
||||
def parse_unicode_sequence(raw_data):
|
||||
"""
|
||||
Parse unicode sequence from raw data.
|
||||
|
||||
:param raw_data: Contains the unicode sequence which needs to parse.
|
||||
:return: The parsed unicode sequence.
|
||||
"""
|
||||
|
||||
result = ''
|
||||
|
||||
for unicode_char in raw_data.split(' '):
|
||||
if unicode_char == '':
|
||||
continue
|
||||
|
||||
# Convert it to unicode code point (from hex value without 0x prefix)
|
||||
result += unichr(int(unicode_char, 16))
|
||||
return result
|
||||
|
||||
|
||||
def read_case_mappings(unicode_data_file, special_casing_file):
|
||||
"""
|
||||
Read the corresponding unicode values of lower and upper case letters and store these in tables.
|
||||
|
||||
:param unicode_data_file: Contains the default case mappings (one-to-one mappings).
|
||||
:param special_casing_file: Contains additional informative case mappings that are either not one-to-one
|
||||
or which are context-sensitive.
|
||||
:return: Upper and lower case mappings.
|
||||
"""
|
||||
|
||||
lower_case_mapping = CaseMapping()
|
||||
upper_case_mapping = CaseMapping()
|
||||
|
||||
# Add one-to-one mappings
|
||||
with open(unicode_data_file) as unicode_data:
|
||||
unicode_data_reader = csv.reader(unicode_data, delimiter=';')
|
||||
|
||||
for line in unicode_data_reader:
|
||||
letter_id = int(line[0], 16)
|
||||
|
||||
# Skip supplementary planes and ascii chars
|
||||
if letter_id >= 0x10000 or letter_id < 128:
|
||||
continue
|
||||
|
||||
capital_letter = line[12]
|
||||
small_letter = line[13]
|
||||
|
||||
if capital_letter:
|
||||
upper_case_mapping.add(letter_id, parse_unicode_sequence(capital_letter))
|
||||
|
||||
if small_letter:
|
||||
lower_case_mapping.add(letter_id, parse_unicode_sequence(small_letter))
|
||||
|
||||
# Update the conversion tables with the special cases
|
||||
with open(special_casing_file) as special_casing:
|
||||
special_casing_reader = csv.reader(special_casing, delimiter=';')
|
||||
|
||||
for line in special_casing_reader:
|
||||
# Skip comment sections and empty lines
|
||||
if not line or line[0].startswith('#'):
|
||||
continue
|
||||
|
||||
# Replace '#' character with empty string
|
||||
for idx, i in enumerate(line):
|
||||
if i.find('#') >= 0:
|
||||
line[idx] = ''
|
||||
|
||||
letter_id = int(line[0], 16)
|
||||
condition_list = line[4]
|
||||
|
||||
# Skip supplementary planes, ascii chars, and condition_list
|
||||
if letter_id >= 0x10000 or letter_id < 128 or condition_list:
|
||||
continue
|
||||
|
||||
small_letter = parse_unicode_sequence(line[1])
|
||||
capital_letter = parse_unicode_sequence(line[3])
|
||||
|
||||
lower_case_mapping.add(letter_id, small_letter)
|
||||
upper_case_mapping.add(letter_id, capital_letter)
|
||||
|
||||
return lower_case_mapping, upper_case_mapping
|
||||
|
||||
|
||||
class CaseMapping(dict):
|
||||
"""Class defines an informative, default mapping."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the case mapping table."""
|
||||
self._conversion_table = {}
|
||||
|
||||
def add(self, letter_id, mapped_value):
|
||||
"""
|
||||
Add mapped value of the unicode letter.
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:param mapped_value: Corresponding character of the case type.
|
||||
"""
|
||||
self._conversion_table[letter_id] = mapped_value
|
||||
|
||||
def remove(self, letter_id):
|
||||
"""
|
||||
Remove mapping from the conversion table.
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
"""
|
||||
del self._conversion_table[letter_id]
|
||||
|
||||
def get_value(self, letter_id):
|
||||
"""
|
||||
Get the mapped value of the given unicode character.
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:return: The mapped value of the character.
|
||||
"""
|
||||
|
||||
if self.contains(letter_id):
|
||||
return self._conversion_table[letter_id]
|
||||
|
||||
return None
|
||||
|
||||
def get_conversion_distance(self, letter_id):
|
||||
"""
|
||||
Calculate the distance between the unicode character and its mapped value
|
||||
(only needs and works with one-to-one mappings).
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:return: The conversion distance.
|
||||
"""
|
||||
|
||||
mapped_value = self.get_value(letter_id)
|
||||
|
||||
if mapped_value and len(mapped_value) == 1:
|
||||
return ord(mapped_value) - letter_id
|
||||
|
||||
return None
|
||||
|
||||
def is_bidirectional_conversion(self, letter_id, other_case_mapping):
|
||||
"""
|
||||
Check that two unicode value are also a mapping value of each other.
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:param other_case_mapping: Comparable case mapping table which possible contains
|
||||
the return direction of the conversion.
|
||||
:return: True, if it's a reverible conversion, false otherwise.
|
||||
"""
|
||||
|
||||
if not self.contains(letter_id):
|
||||
return False
|
||||
|
||||
# Check one-to-one mapping
|
||||
mapped_value = self.get_value(letter_id)
|
||||
if len(mapped_value) > 1:
|
||||
return False
|
||||
|
||||
# Check two way conversions
|
||||
mapped_value_id = ord(mapped_value)
|
||||
if other_case_mapping.get_value(mapped_value_id) != unichr(letter_id):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def contains(self, letter_id):
|
||||
"""
|
||||
Check that a unicode character is in the conversion table.
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:return: True, if it contains the character, false otherwise.
|
||||
"""
|
||||
if letter_id in self._conversion_table:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def get_table(self):
|
||||
return self._conversion_table
|
||||
|
||||
def extract_ranges(self, other_case_mapping=None):
|
||||
"""
|
||||
Extract ranges from case mappings
|
||||
(the second param is optional, if it's not empty, a range will contains bidirectional conversions only).
|
||||
|
||||
:param letter_id: An integer, representing the unicode code point of the character.
|
||||
:param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
|
||||
:return: A table with the start points and their mapped value, and another table with the lengths of the ranges.
|
||||
"""
|
||||
|
||||
in_range = False
|
||||
range_position = -1
|
||||
ranges = []
|
||||
range_lengths = []
|
||||
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
prev_letter_id = letter_id - 1
|
||||
|
||||
# One-way conversions
|
||||
if other_case_mapping is None:
|
||||
if len(self.get_value(letter_id)) > 1:
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
if not self.contains(prev_letter_id) or len(self.get_value(prev_letter_id)) > 1:
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
# Two way conversions
|
||||
else:
|
||||
if not self.is_bidirectional_conversion(letter_id, other_case_mapping):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
conv_distance = self.get_conversion_distance(letter_id)
|
||||
prev_conv_distance = self.get_conversion_distance(prev_letter_id)
|
||||
|
||||
if (conv_distance != prev_conv_distance):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
if in_range:
|
||||
range_lengths[range_position] += 1
|
||||
else:
|
||||
in_range = True
|
||||
range_position += 1
|
||||
|
||||
# Add the start point of the range and its mapped value
|
||||
ranges.extend([prev_letter_id, ord(self.get_value(prev_letter_id))])
|
||||
range_lengths.append(2)
|
||||
|
||||
# Remove all ranges from the case mapping table.
|
||||
index = 0
|
||||
while index != len(ranges):
|
||||
range_length = range_lengths[index // 2]
|
||||
|
||||
for incr in range(range_length):
|
||||
self.remove(ranges[index] + incr)
|
||||
if other_case_mapping is not None:
|
||||
other_case_mapping.remove(ranges[index + 1] + incr)
|
||||
|
||||
index += 2
|
||||
|
||||
return ranges, range_lengths
|
||||
|
||||
def extract_character_pair_ranges(self, other_case_mapping):
|
||||
"""
|
||||
Extract two or more character pairs from the case mapping tables.
|
||||
|
||||
:param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
|
||||
:return: A table with the start points, and another table with the lengths of the ranges.
|
||||
"""
|
||||
|
||||
start_points = []
|
||||
lengths = []
|
||||
in_range = False
|
||||
element_counter = -1
|
||||
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
# Only extract character pairs
|
||||
if not self.is_bidirectional_conversion(letter_id, other_case_mapping):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
if self.get_value(letter_id) == unichr(letter_id + 1):
|
||||
prev_letter_id = letter_id - 2
|
||||
|
||||
if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping):
|
||||
in_range = False
|
||||
|
||||
if in_range:
|
||||
lengths[element_counter] += 2
|
||||
else:
|
||||
element_counter += 1
|
||||
start_points.append(letter_id)
|
||||
lengths.append(2)
|
||||
in_range = True
|
||||
|
||||
else:
|
||||
in_range = False
|
||||
|
||||
# Remove all founded case mapping from the conversion tables after the scanning method
|
||||
idx = 0
|
||||
while idx != len(start_points):
|
||||
letter_id = start_points[idx]
|
||||
conv_length = lengths[idx]
|
||||
|
||||
for incr in range(0, conv_length, 2):
|
||||
self.remove(letter_id + incr)
|
||||
other_case_mapping.remove(letter_id + 1 + incr)
|
||||
|
||||
idx += 1
|
||||
|
||||
return start_points, lengths
|
||||
|
||||
def extract_character_pairs(self, other_case_mapping):
|
||||
"""
|
||||
Extract character pairs. Check that two unicode value are also a mapping value of each other.
|
||||
|
||||
:param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
|
||||
:return: A table with character pairs.
|
||||
"""
|
||||
|
||||
character_pairs = []
|
||||
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
if self.is_bidirectional_conversion(letter_id, other_case_mapping):
|
||||
mapped_value = self.get_value(letter_id)
|
||||
character_pairs.extend([letter_id, ord(mapped_value)])
|
||||
|
||||
# Remove character pairs from case mapping tables
|
||||
self.remove(letter_id)
|
||||
other_case_mapping.remove(ord(mapped_value))
|
||||
|
||||
return character_pairs
|
||||
|
||||
def extract_special_ranges(self):
|
||||
"""
|
||||
Extract special ranges. It contains that ranges of one-to-two mappings where the second character
|
||||
of the mapped values are equals and the other characters are following each other.
|
||||
eg.: \u1f80 and \u1f81 will be in one range becase their upper-case values are \u1f08\u0399 and \u1f09\u0399
|
||||
|
||||
:return: A table with the start points and their mapped values, and a table with the lengths of the ranges.
|
||||
"""
|
||||
|
||||
special_ranges = []
|
||||
special_range_lengths = []
|
||||
|
||||
range_position = -1
|
||||
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
mapped_value = self.get_value(letter_id)
|
||||
|
||||
if len(mapped_value) != 2:
|
||||
continue
|
||||
|
||||
prev_letter_id = letter_id - 1
|
||||
|
||||
if not self.contains(prev_letter_id):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
prev_mapped_value = self.get_value(prev_letter_id)
|
||||
|
||||
if len(prev_mapped_value) != 2:
|
||||
continue
|
||||
|
||||
if prev_mapped_value[1] != mapped_value[1]:
|
||||
continue
|
||||
|
||||
if (ord(prev_mapped_value[0]) - prev_letter_id) != (ord(mapped_value[0]) - letter_id):
|
||||
in_range = False
|
||||
continue
|
||||
|
||||
if in_range:
|
||||
special_range_lengths[range_position] += 1
|
||||
else:
|
||||
range_position += 1
|
||||
in_range = True
|
||||
|
||||
special_ranges.extend([prev_letter_id, ord(prev_mapped_value[0]), ord(prev_mapped_value[1])])
|
||||
special_range_lengths.append(1)
|
||||
|
||||
# Remove special ranges from the conversion table
|
||||
idx = 0
|
||||
while idx != len(special_ranges):
|
||||
range_length = special_range_lengths[idx // 3]
|
||||
letter_id = special_ranges[idx]
|
||||
|
||||
for incr in range(range_length):
|
||||
self.remove(special_ranges[idx] + incr)
|
||||
|
||||
idx += 3
|
||||
|
||||
return special_ranges, special_range_lengths
|
||||
|
||||
def extract_conversions(self):
|
||||
"""
|
||||
Extract conversions. It provide the full (or remained) case mappings from the table.
|
||||
The counter table contains the information of how much one-to-one, one-to-two or one-to-three mappings
|
||||
exists successively in the conversion table.
|
||||
|
||||
:return: A table with conversions, and a table with counters.
|
||||
"""
|
||||
|
||||
unicodes = [[], [], []]
|
||||
unicode_lengths = [0, 0, 0]
|
||||
|
||||
# 1 to 1 byte
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
mapped_value = self.get_value(letter_id)
|
||||
|
||||
if len(mapped_value) != 1:
|
||||
continue
|
||||
|
||||
unicodes[0].extend([letter_id, ord(mapped_value)])
|
||||
self.remove(letter_id)
|
||||
|
||||
# 1 to 2 bytes
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
mapped_value = self.get_value(letter_id)
|
||||
|
||||
if len(mapped_value) != 2:
|
||||
continue
|
||||
|
||||
unicodes[1].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1])])
|
||||
self.remove(letter_id)
|
||||
|
||||
# 1 to 3 bytes
|
||||
for letter_id in sorted(self._conversion_table.keys()):
|
||||
mapped_value = self.get_value(letter_id)
|
||||
|
||||
if len(mapped_value) != 3:
|
||||
continue
|
||||
|
||||
unicodes[2].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1]), ord(mapped_value[2])])
|
||||
self.remove(letter_id)
|
||||
|
||||
unicode_lengths = [int(len(unicodes[0]) / 2), int(len(unicodes[1]) / 3), int(len(unicodes[2]) / 4)]
|
||||
|
||||
return list(itertools.chain.from_iterable(unicodes)), unicode_lengths
|
||||
|
||||
|
||||
def regroup(l, n):
|
||||
return [l[i:i+n] for i in range(0, len(l), n)]
|
||||
|
||||
|
||||
def hex_format(ch):
|
||||
if isinstance(ch, str):
|
||||
ch = ord(ch)
|
||||
|
||||
return "0x{:04x}".format(ch)
|
||||
|
||||
|
||||
def format_code(code, indent):
|
||||
lines = []
|
||||
# convert all characters to hex format
|
||||
converted_code = map(hex_format, code)
|
||||
# 10 hex number per line
|
||||
for line in regroup(", ".join(converted_code), 10 * 8):
|
||||
lines.append((' ' * indent) + line.strip())
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def create_c_format_table(type_name, array_name, table, description=""):
|
||||
return """{DESC}
|
||||
static const {TYPE} jerry_{NAME}[] JERRY_CONST_DATA =
|
||||
{{
|
||||
{TABLE}
|
||||
}};
|
||||
|
||||
""".format(DESC=description, TYPE=type_name, NAME=array_name, TABLE=format_code(table, 1))
|
||||
|
||||
|
||||
def copy_tables_to_c_source(gen_tables, c_source):
|
||||
data = []
|
||||
|
||||
header = """/* Copyright JS Foundation and other contributors, http://js.foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
* This file is automatically generated by the {SCRIPT} script. Do not edit!
|
||||
*/
|
||||
|
||||
""".format(SCRIPT=os.path.basename(__file__))
|
||||
|
||||
data.append(header)
|
||||
|
||||
character_case_ranges = gen_tables.get_character_case_ranges()
|
||||
character_pair_ranges = gen_tables.get_character_pair_ranges()
|
||||
character_pairs = gen_tables.get_character_pairs()
|
||||
upper_case_special_ranges = gen_tables.get_upper_case_special_ranges()
|
||||
lower_case_ranges = gen_tables.get_lower_case_ranges()
|
||||
lower_case_conversions = gen_tables.get_lower_case_conversions()
|
||||
upper_case_conversions = gen_tables.get_upper_case_conversions()
|
||||
|
||||
description = "/* Contains start points of character case ranges (these are bidirectional conversions). */"
|
||||
data.append(create_c_format_table('uint16_t', 'character_case_ranges',
|
||||
character_case_ranges[0],
|
||||
description))
|
||||
|
||||
description = "/* Interval lengths of start points in `character_case_ranges` table. */"
|
||||
data.append(create_c_format_table('uint8_t',
|
||||
'character_case_range_lengths',
|
||||
character_case_ranges[1],
|
||||
description))
|
||||
|
||||
description = "/* Contains the start points of bidirectional conversion ranges. */"
|
||||
data.append(create_c_format_table('uint16_t',
|
||||
'character_pair_ranges',
|
||||
character_pair_ranges[0],
|
||||
description))
|
||||
|
||||
description = "/* Interval lengths of start points in `character_pair_ranges` table. */"
|
||||
data.append(create_c_format_table('uint8_t',
|
||||
'character_pair_range_lengths',
|
||||
character_pair_ranges[1],
|
||||
description))
|
||||
|
||||
description = "/* Contains lower/upper case bidirectional conversion pairs. */"
|
||||
data.append(create_c_format_table('uint16_t',
|
||||
'character_pairs',
|
||||
character_pairs,
|
||||
description))
|
||||
|
||||
description = """/* Contains start points of one-to-two uppercase ranges where the second character
|
||||
* is always the same.
|
||||
*/"""
|
||||
data.append(create_c_format_table('uint16_t',
|
||||
'upper_case_special_ranges',
|
||||
upper_case_special_ranges[0],
|
||||
description))
|
||||
|
||||
description = "/* Interval lengths for start points in `upper_case_special_ranges` table. */"
|
||||
data.append(create_c_format_table('uint8_t',
|
||||
'upper_case_special_range_lengths',
|
||||
upper_case_special_ranges[1],
|
||||
description))
|
||||
|
||||
description = "/* Contains start points of lowercase ranges. */"
|
||||
data.append(create_c_format_table('uint16_t', 'lower_case_ranges', lower_case_ranges[0], description))
|
||||
|
||||
description = "/* Interval lengths for start points in `lower_case_ranges` table. */"
|
||||
data.append(create_c_format_table('uint8_t', 'lower_case_range_lengths', lower_case_ranges[1], description))
|
||||
|
||||
description = "/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */"
|
||||
data.append(create_c_format_table('uint16_t',
|
||||
'lower_case_conversions',
|
||||
lower_case_conversions[0],
|
||||
description))
|
||||
|
||||
description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */"
|
||||
|
||||
data.append(create_c_format_table('uint8_t',
|
||||
'lower_case_conversion_counters',
|
||||
lower_case_conversions[1],
|
||||
description))
|
||||
|
||||
description = "/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */"
|
||||
data.append(create_c_format_table('uint16_t',
|
||||
'upper_case_conversions',
|
||||
upper_case_conversions[0],
|
||||
description))
|
||||
|
||||
description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */"
|
||||
data.append(create_c_format_table('uint8_t',
|
||||
'upper_case_conversion_counters',
|
||||
upper_case_conversions[1],
|
||||
description))
|
||||
|
||||
with open(c_source, 'w') as genereted_source:
|
||||
genereted_source.write(''.join(data))
|
||||
|
||||
|
||||
class GenTables(object):
|
||||
"""Class defines an informative, default generated tables."""
|
||||
|
||||
def __init__(self, lower_case_table, upper_case_table):
|
||||
"""
|
||||
Generate the extracted tables from the given case mapping tables.
|
||||
|
||||
:param lower_case_table: Lower-case mappings.
|
||||
:param upper_case_table: Upper-case mappings.
|
||||
"""
|
||||
|
||||
self._character_case_ranges = lower_case_table.extract_ranges(upper_case_table)
|
||||
self._character_pair_ranges = lower_case_table.extract_character_pair_ranges(upper_case_table)
|
||||
self._character_pairs = lower_case_table.extract_character_pairs(upper_case_table)
|
||||
self._upper_case_special_ranges = upper_case_table.extract_special_ranges()
|
||||
self._lower_case_ranges = lower_case_table.extract_ranges()
|
||||
self._lower_case_conversions = lower_case_table.extract_conversions()
|
||||
self._upper_case_conversions = upper_case_table.extract_conversions()
|
||||
|
||||
if lower_case_table.get_table():
|
||||
warnings.warn('Not all elements extracted from the lowercase conversion table!')
|
||||
if upper_case_table.get_table():
|
||||
warnings.warn('Not all elements extracted from the uppercase conversion table!')
|
||||
|
||||
def get_character_case_ranges(self):
|
||||
return self._character_case_ranges
|
||||
|
||||
def get_character_pair_ranges(self):
|
||||
return self._character_pair_ranges
|
||||
|
||||
def get_character_pairs(self):
|
||||
return self._character_pairs
|
||||
|
||||
def get_upper_case_special_ranges(self):
|
||||
return self._upper_case_special_ranges
|
||||
|
||||
def get_lower_case_ranges(self):
|
||||
return self._lower_case_ranges
|
||||
|
||||
def get_lower_case_conversions(self):
|
||||
return self._lower_case_conversions
|
||||
|
||||
def get_upper_case_conversions(self):
|
||||
return self._upper_case_conversions
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument('--unicode-data',
|
||||
metavar='FILE',
|
||||
action='store',
|
||||
required=True,
|
||||
help='specify the unicode data file')
|
||||
|
||||
parser.add_argument('--special-casing',
|
||||
metavar='FILE',
|
||||
action='store',
|
||||
required=True,
|
||||
help='specify the special casing file')
|
||||
|
||||
parser.add_argument('--c-source',
|
||||
metavar='FILE',
|
||||
action='store',
|
||||
default=C_SOURCE_FILE,
|
||||
help='specify the output c source (default: %(default)s)')
|
||||
|
||||
script_args = parser.parse_args()
|
||||
|
||||
if not os.path.isfile(script_args.unicode_data) or not os.access(script_args.unicode_data, os.R_OK):
|
||||
print('The %s file is missing or not readable!' % script_args.unicode_data)
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.isfile(script_args.special_casing) or not os.access(script_args.special_casing, os.R_OK):
|
||||
print('The %s file is missing or not readable!' % script_args.special_casing)
|
||||
sys.exit(1)
|
||||
|
||||
lower_case_table, upper_case_table = read_case_mappings(script_args.unicode_data, script_args.special_casing)
|
||||
|
||||
gen_tables = GenTables(lower_case_table, upper_case_table)
|
||||
|
||||
copy_tables_to_c_source(gen_tables, script_args.c_source)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user