From 9cbd5e78ada97ac1110aabde9354c3304e8cbbf5 Mon Sep 17 00:00:00 2001 From: Robert Sipka Date: Wed, 19 Apr 2017 00:08:32 +0200 Subject: [PATCH] Refactor js2c.py and eliminate related pylint warnings (#1728) JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com --- targets/esp8266/Makefile.esp8266 | 2 +- targets/mbed/Makefile.mbed | 2 +- targets/mbedos5/Makefile | 2 +- targets/tools/js2c.py | 154 ------------------ ...unicode_c_source.py => c_source_helper.py} | 26 +-- tools/js2c.py | 124 ++++++++++++++ tools/unicode_case_conversion.py | 10 +- tools/unicode_ranges.py | 11 +- 8 files changed, 152 insertions(+), 179 deletions(-) delete mode 100755 targets/tools/js2c.py rename tools/{unicode_c_source.py => c_source_helper.py} (76%) create mode 100755 tools/js2c.py diff --git a/targets/esp8266/Makefile.esp8266 b/targets/esp8266/Makefile.esp8266 index 3705f3013..2ac0213e9 100644 --- a/targets/esp8266/Makefile.esp8266 +++ b/targets/esp8266/Makefile.esp8266 @@ -92,7 +92,7 @@ jerry: js2c: - cd targets/esp8266; ../tools/js2c.py + cd targets/esp8266; ../../tools/js2c.py mkbin: diff --git a/targets/mbed/Makefile.mbed b/targets/mbed/Makefile.mbed index e3bd1434a..ea2b59b6c 100644 --- a/targets/mbed/Makefile.mbed +++ b/targets/mbed/Makefile.mbed @@ -71,7 +71,7 @@ jerry: cp $(BUILD_DIR)/lib/libjerry-libm.a $(COPYTARGET)/libjerrylibm.a js2c: - cd targets/mbed; ../tools/js2c.py; + cd targets/mbed; ../../tools/js2c.py; yotta: cd targets/mbed; \ diff --git a/targets/mbedos5/Makefile b/targets/mbedos5/Makefile index dce65768a..9ac853724 100644 --- a/targets/mbedos5/Makefile +++ b/targets/mbedos5/Makefile @@ -66,7 +66,7 @@ clean: rm -rf .build/$(BOARD) js2c: js/main.js js/flash_leds.js - python ../tools/js2c.py --ignore pins.js + python ../../tools/js2c.py --ignore pins.js source/pins.cpp: python tools/generate_pins.py ${BOARD} diff --git a/targets/tools/js2c.py b/targets/tools/js2c.py deleted file mode 100755 index 7ab494937..000000000 --- a/targets/tools/js2c.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python - -# Copyright JS Foundation and other contributors, http://js.foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This file converts ./js/*.js to a C-array in ./source/jerry-targetjs.h file - -import argparse -import glob -import os -import re -import sys - -special_chars = re.compile(r'[-\\?\'".]') - -def extractName(path): - return special_chars.sub('_', os.path.splitext(os.path.basename(path))[0]) - -def writeLine(fo, content, indent=0): - buf = ' ' * indent + content + '\n' - fo.write(buf) - -def regroup(l, n): - return [ l[i:i+n] for i in range(0, len(l), n) ] - -def removeComments(code): - pattern = r'(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)' - regex = re.compile(pattern, re.MULTILINE | re.DOTALL) - def _replacer(match): - if match.group(2) is not None: - return "" - else: - return match.group(1) - return regex.sub(_replacer, code) - -def removeWhitespaces(code): - return re.sub('\n+', '\n', re.sub('\n +', '\n', code)) - - -LICENSE = '''/* Copyright JS Foundation and other contributors, http://js.foundation - * - * Licensed under the Apache License, Version 2.0 (the \"License\"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an \"AS IS\" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This file is generated by js2c.py. Please do not modify. - */ - -''' - -HEADER = '''#ifndef JERRY_TARGETJS_H -#define JERRY_TARGETJS_H - -''' - -FOOTER = ''' -#endif - -''' - -parser = argparse.ArgumentParser(description="js2c") -parser.add_argument('--build-type', help='build type', default='release', choices=['release', 'debug']) -parser.add_argument('--ignore', help='files to ignore', dest='ignore_files', default=[], action='append') -parser.add_argument('--no-main', help="don't require a 'main.js' file", dest='main', action='store_false', default=True) -parser.add_argument('--js-source', dest='js_source_path', default='./js', help='Source directory of JavaScript files" (default: %(default)s)') -parser.add_argument('--dest', dest='output_path', default='./source', help="Destination directory of 'jerry-targetjs.h' (default: %(default)s)") - -args = parser.parse_args() - -# argument processing -build_type = args.build_type -ignore_files = args.ignore_files - -fout = open(os.path.join(args.output_path, 'jerry-targetjs.h'), 'w') -fout.write(LICENSE); -fout.write(HEADER); - -def exportOneFile(path, name): - fout.write('const static char ' + name + '_n[] = "' + name + '";\n') - fout.write('const static char ' + name + '_s[] =\n{\n') - - fin = open(path, 'r'); - code = fin.read() + '\0' - - # minimize code when release mode - if build_type != 'debug': - code = removeComments(code) - code = removeWhitespaces(code) - - for line in regroup(code, 10): - buf = ', '.join(map(lambda ch: format(ord(ch),"#04x"), line)) - if line[-1] != '\0': - buf += ',' - writeLine(fout, buf, 1) - writeLine(fout, '};') - writeLine(fout, 'const static int ' + name + '_l = ' + str(len(code)-1) + ';') - writeLine(fout, '') - - fin.close(); - -def exportOneName(name): - writeLine(fout, '{ ' + name + '_n, ' + name + '_s, ' + name + '_l }, \\', 1) - -files = glob.glob(os.path.join(args.js_source_path, '*.js')) -for path in files: - name = extractName(path) - if os.path.basename(path) not in ignore_files: - exportOneFile(path, name) - - -NATIVE_STRUCT = ''' -struct js_source_all { - const char* name; - const char* source; - const int length; -}; - -#define DECLARE_JS_CODES \\ -struct js_source_all js_codes[] = \\ -{ \\ -''' - -fout.write(NATIVE_STRUCT) -if args.main: - exportOneName('main') -for filename in files: - name = extractName(filename) - if name != 'main' and os.path.basename(filename) not in ignore_files: - exportOneName(name) - -writeLine(fout, '{ NULL, NULL, 0 } \\', 1) -writeLine(fout, '};') - -fout.write(FOOTER) -fout.close() diff --git a/tools/unicode_c_source.py b/tools/c_source_helper.py similarity index 76% rename from tools/unicode_c_source.py rename to tools/c_source_helper.py index 7c5d66ba9..89c18c088 100644 --- a/tools/unicode_c_source.py +++ b/tools/c_source_helper.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + LICENSE = """/* Copyright JS Foundation and other contributors, http://js.foundation * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +31,7 @@ LICENSE = """/* Copyright JS Foundation and other contributors, http://js.founda */""" -class Source(object): +class UniCodeSource(object): def __init__(self, filepath): self.__filepath = filepath self.__header = [LICENSE, ""] @@ -38,7 +39,7 @@ class Source(object): def complete_header(self, completion): self.__header.append(completion) - self.__header.append("") # for an extra empty line + self.__header.append("") # for an extra empty line def add_table(self, table, table_name, table_type, table_descr): self.__data.append(table_descr) @@ -46,30 +47,33 @@ class Source(object): self.__data.append("{") self.__data.append(format_code(table, 1)) self.__data.append("};") - self.__data.append("") # for an extra empty line + self.__data.append("") # for an extra empty line def generate(self): - with open(self.__filepath, 'w') as genereted_source: - genereted_source.write("\n".join(self.__header)) - genereted_source.write("\n".join(self.__data)) + with open(self.__filepath, 'w') as generated_source: + generated_source.write("\n".join(self.__header)) + generated_source.write("\n".join(self.__data)) def regroup(list_to_group, num): return [list_to_group[i:i+num] for i in range(0, len(list_to_group), num)] -def hex_format(char): +def hex_format(char, digit_number): if isinstance(char, str): char = ord(char) - return "0x{:04x}".format(char) + return ("0x{:0%sx}" % digit_number).format(char) -def format_code(code, indent): +def format_code(code, indent, digit_number=4): lines = [] + + nums_per_line = 10 + width = nums_per_line * (digit_number + 4) # convert all characters to hex format - converted_code = [hex_format(char) for char in code] + converted_code = [hex_format(char, digit_number) for char in code] # 10 hex number per line - for line in regroup(", ".join(converted_code), 10 * 8): + for line in regroup(", ".join(converted_code), width): lines.append((' ' * indent) + line.strip()) return "\n".join(lines) diff --git a/tools/js2c.py b/tools/js2c.py new file mode 100755 index 000000000..63a66042f --- /dev/null +++ b/tools/js2c.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +# Copyright JS Foundation and other contributors, http://js.foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file converts ./js/*.js to a C-array in ./source/jerry-targetjs.h file + +import argparse +import glob +import os +import re +import c_source_helper + +HEADER = '''#ifndef JERRY_TARGETJS_H +#define JERRY_TARGETJS_H +''' + +FOOTER = ''' +#endif +''' + +NATIVE_STRUCT = ''' +struct js_source_all { + const char* name; + const char* source; + const int length; +}; + +#define DECLARE_JS_CODES \\ +struct js_source_all js_codes[] = \\ +{ \\''' + + +def extract_name(path): + special_chars = re.compile(r'[-\\?\'".]') + return special_chars.sub('_', os.path.splitext(os.path.basename(path))[0]) + + +def reduce_code(code): + code = re.sub(r"/\*.*?\*/", "", code, flags=re.DOTALL) # remove all occurance streamed comments + code = re.sub(r"//.*?\n", "", code) # remove all occurance singleline comments + code = re.sub('\n+', '\n', re.sub('\n +', '\n', code)) # remove white spaces + return code + + +def js_to_native_code(path, name, build_type): + with open(path, 'r') as js_source: + code = js_source.read() + + if build_type != 'debug': + code = reduce_code(code) + + data = c_source_helper.format_code(code, 1, 2) + + native_code = """const static char {0}_n[] = "{0}"; +const static char {0}_s[] = +{{ +{1} +}}; +const static int {0}_l = {2}; +""".format(name, data, len(code)) + + return native_code + + +def main(): + parser = argparse.ArgumentParser(description="js2c") + parser.add_argument('--build-type', help='build type', default='release', choices=['release', 'debug']) + parser.add_argument('--ignore', help='files to ignore', dest='ignore_files', default=[], action='append') + parser.add_argument('--no-main', + help="don't require a 'main.js' file", + dest='main', + action='store_false', + default=True) + parser.add_argument('--js-source', + dest='js_source_path', + default='./js', + help='Source directory of JavaScript files" (default: %(default)s)') + parser.add_argument('--dest', + dest='output_path', + default='./source', + help="Destination directory of 'jerry-targetjs.h' (default: %(default)s)") + + script_args = parser.parse_args() + + gen_line = "/* This file is generated by %s. Please do not modify. */" % os.path.basename(__file__) + + gen_output = [c_source_helper.LICENSE, "", gen_line, "", HEADER] + gen_structs = [NATIVE_STRUCT] + + if script_args.main: + gen_structs.append(' {{ {0}_n, {0}_s, {0}_l }}, \\'.format("main")) + + files = glob.glob(os.path.join(script_args.js_source_path, '*.js')) + + for path in files: + if os.path.basename(path) not in script_args.ignore_files: + name = extract_name(path) + gen_output.append(js_to_native_code(path, name, script_args.build_type)) + if name != 'main': + gen_structs.append(' {{ {0}_n, {0}_s, {0}_l }}, \\'.format(name)) + + gen_structs.append(' { NULL, NULL, 0 } \\\n};') + + gen_output.append("\n".join(gen_structs)) + gen_output.append(FOOTER) + + with open(os.path.join(script_args.output_path, 'jerry-targetjs.h'), 'w') as gen_file: + gen_file.write("\n".join(gen_output)) + + +if __name__ == "__main__": + main() diff --git a/tools/unicode_case_conversion.py b/tools/unicode_case_conversion.py index 9f90846ac..2dd19cc45 100755 --- a/tools/unicode_case_conversion.py +++ b/tools/unicode_case_conversion.py @@ -15,8 +15,6 @@ # limitations under the License. from __future__ import print_function -from settings import PROJECT_DIR -from unicode_c_source import Source import argparse import csv @@ -25,6 +23,9 @@ import os import sys import warnings +from settings import PROJECT_DIR +from c_source_helper import UniCodeSource + CONVERSIONS_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h') @@ -69,7 +70,7 @@ def main(): lower_case_conversions = conv_tables.get_lower_case_conversions() upper_case_conversions = conv_tables.get_upper_case_conversions() - c_source = Source(script_args.c_source) + c_source = UniCodeSource(script_args.c_source) unicode_file = os.path.basename(script_args.unicode_data) spec_casing_file = os.path.basename(script_args.special_casing) @@ -395,8 +396,7 @@ def extract_character_pair_ranges(letter_case, reverse_letter_case): in_range = False # Remove all founded case mapping from the conversion tables after the scanning method - for idx in range(len(start_points)): - letter_id = start_points[idx] + for idx, letter_id in enumerate(start_points): conv_length = lengths[idx] for incr in range(0, conv_length, 2): diff --git a/tools/unicode_ranges.py b/tools/unicode_ranges.py index 5cca6718a..67d806961 100755 --- a/tools/unicode_ranges.py +++ b/tools/unicode_ranges.py @@ -27,8 +27,6 @@ # separators: Zs from __future__ import print_function -from settings import PROJECT_DIR -from unicode_c_source import Source import argparse import bisect @@ -37,6 +35,9 @@ import itertools import os import sys +from c_source_helper import UniCodeSource +from settings import PROJECT_DIR + RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h') def main(): @@ -65,7 +66,7 @@ def main(): non_letter_tables = split_list(list(ranges(non_letters))) separator_tables = split_list(list(ranges(separators))) - c_source = Source(script_args.c_source) + c_source = UniCodeSource(script_args.c_source) header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__), " * from %s. Do not edit! */" % os.path.basename(script_args.unicode_data), @@ -181,9 +182,7 @@ def read_categories(unicode_data_file): separators = [] with open(unicode_data_file) as unicode_data: - unicode_data_reader = csv.reader(unicode_data, delimiter=';') - - for line in unicode_data_reader: + for line in csv.reader(unicode_data, delimiter=';'): unicode_id = int(line[0], 16) # Skip supplementary planes and ascii chars