From fdaacde6671cf0a370f7f5b50a45a489efc6bb2f Mon Sep 17 00:00:00 2001 From: Akos Kiss Date: Mon, 11 Jan 2021 13:07:40 +0100 Subject: [PATCH] Officially introduce amalgamated builds (#4416) Remove redundancy between all-in-one and all-in-one-source builds by keeping only the second, and adopt the more established term "amalgamated" build for it. This change includes the following: - Replace `ENABLE_ALL_IN_ONE` and `ENABLE_ALL_IN_ONE_SOURCE` cmake options with `ENABLE_AMALGAM` top-level option. - Replace `--all-in-one` option of `build.py` helper with `--amalgam`. - Merge the `srcmerger.py` and `srcgenerator.py` tool scripts into `amalgam.py` (with improvements). - Update documentation. JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu --- CMakeLists.txt | 8 +- appveyor.yml | 2 +- docs/01.CONFIGURATION.md | 37 +++-- jerry-core/CMakeLists.txt | 54 ++----- jerry-math/CMakeLists.txt | 23 ++- jerry-port/default/CMakeLists.txt | 23 ++- targets/esp8266/Makefile.esp8266 | 2 +- targets/nuttx-stm32f4/Makefile.travis | 2 +- targets/nuttx-stm32f4/README.md | 2 +- targets/particle/Makefile.particle | 1 - targets/riot-stm32f4/Makefile.riot | 1 - targets/zephyr/Makefile.zephyr | 1 - tools/{srcmerger.py => amalgam.py} | 207 +++++++++++++++++++------- tools/build.py | 6 +- tools/run-tests.py | 6 +- tools/srcgenerator.py | 154 ------------------- 16 files changed, 224 insertions(+), 305 deletions(-) rename tools/{srcmerger.py => amalgam.py} (53%) mode change 100644 => 100755 delete mode 100644 tools/srcgenerator.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 4da5b19f1..c3a992f91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ set(DOCTESTS OFF CACHE BOOL "Build doc tests?") # Optional build settings set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries?") +set(ENABLE_AMALGAM OFF CACHE BOOL "Enable amalgamated build?") set(ENABLE_LTO ON CACHE BOOL "Enable LTO build?") set(ENABLE_STRIP ON CACHE BOOL "Enable stripping all symbols from release binary?") @@ -104,8 +105,10 @@ if(USING_TI) endif() if(USING_MSVC) + set(ENABLE_AMALGAM ON) # FIXME: This should not be needed but right now it is. To be tracked down and followed up. set(ENABLE_STRIP OFF) + set(ENABLE_AMALGAM_MESSAGE " (FORCED BY COMPILER)") set(ENABLE_STRIP_MESSAGE " (FORCED BY COMPILER)") endif() @@ -121,6 +124,7 @@ message(STATUS "CMAKE_C_COMPILER_ID " ${CMAKE_C_COMPILER_ID}) message(STATUS "CMAKE_SYSTEM_NAME " ${CMAKE_SYSTEM_NAME}) message(STATUS "CMAKE_SYSTEM_PROCESSOR " ${CMAKE_SYSTEM_PROCESSOR}) message(STATUS "BUILD_SHARED_LIBS " ${BUILD_SHARED_LIBS}) +message(STATUS "ENABLE_AMALGAM " ${ENABLE_AMALGAM} ${ENABLE_AMALGAM_MESSAGE}) message(STATUS "ENABLE_LTO " ${ENABLE_LTO} ${ENABLE_LTO_MESSAGE}) message(STATUS "ENABLE_STRIP " ${ENABLE_STRIP} ${ENABLE_STRIP_MESSAGE}) message(STATUS "JERRY_VERSION " ${JERRY_VERSION}) @@ -247,8 +251,8 @@ if(DEFINED EXTERNAL_LINKER_FLAGS) jerry_add_link_flags(${EXTERNAL_LINKER_FLAGS}) endif() -# Used for placeholder to attach single source build targets -add_custom_target(generate-single-source) +# Used as placeholder to attach amalgamated build targets to +add_custom_target(amalgam) # Jerry's libm if(JERRY_MATH) diff --git a/appveyor.yml b/appveyor.yml index ac88bbcf8..cc215ac00 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,5 +23,5 @@ before_build: - if "%PLATFORM%"=="x64" cmake -G"Visual Studio 15 2017 Win64" -Bbuild -H. -DJERRY_DEBUGGER=ON build: project: build\Jerry.sln - parallel: true + parallel: false # FIXME: This should not be needed but right now it is: msbuild generates all amalgamated files twice, at the same time in parallel builds, leading to I/O errors. verbosity: minimal diff --git a/docs/01.CONFIGURATION.md b/docs/01.CONFIGURATION.md index 962a69c44..ebb20e4ad 100644 --- a/docs/01.CONFIGURATION.md +++ b/docs/01.CONFIGURATION.md @@ -5,16 +5,19 @@ A configuration option's value can be changed either by providing specific C pre This document lists the available configuration options, shows the configuration name for C, CMake, and python, and provides a brief description that explains the effect of the options. -### All-in-one build +### Amalgamated build -Enables the All-in-one build process, which aggregates the contents of each source file, and uses this combined file to compile the JerryScript library. -This process can provide comparable results to link time optimization, and can be useful when LTO is not available otherwise. +Enables the amalgamated build process, which aggregates the contents of each source file per library +(i.e., JerryScript's core, port, and math libraries), and uses these combined files to compile the project. +This process can provide comparable results to link-time optimization, and can be useful when LTO is not available otherwise. | Options | | |---------|----------------------------------------------| | C: | `` | -| CMake: | `-DENABLE_ALL_IN_ONE=ON/OFF` | -| Python: | `--all-in-one=ON/OFF` | +| CMake: | `-DENABLE_AMALGAM=ON/OFF` | +| Python: | `--amalgam=ON/OFF` | + +See also: [Amalgamated sources](#amalgamated-sources) ### Jerry debugger @@ -278,18 +281,20 @@ This option is disabled by default. | Python: | `--mem-stress-test=ON/OFF` | -# Single source build mode +# Amalgamated sources -There is a special mode to use/"build" JerryScript. That is generating a single C file which can be -included into projects quickly. To achive this the following command can be executed to create -a set of files into the `gen_src` directory (Note that the command is executed in the jerryscript root directory -but can be adapted to run outside of the project root dir): +The sources of JerryScript can be combined into a handful of files to allow their easy integration +in other projects. To achieve this, the following command can be executed to create a set of files +into the `amalgam` directory: ```sh -$ python tools/srcgenerator.py --output-dir gen_src --jerry-core --jerry-port-default --jerry-math +$ python tools/amalgam.py --output-dir amalgam --jerry-core --jerry-port-default --jerry-math ``` -The command creates the following files in the `gen_src` dir: +(Note: In the example above, the command is executed from the project's root directory, but that is +not mandatory.) + +The command creates the following files in the `amalgam` dir: * `jerryscript.c` * `jerryscript.h` @@ -304,15 +309,15 @@ should be adapted to the required use-case. See the file contents for more detai default configuration. (Note: This config file is created from the the `jerry-core/config.h` file.) These files can be directly compiled with an application using the JerryScript API. -For example with the following command: +E.g., using a command similar to the one below: ```sh -$ gcc -Wall -o demo_app demo_app.c gen_src/jerryscript.c gen_src/jerryscript-port-default.c jerryscript-math.c -Igen_src/ +$ gcc -Wall -o demo_app demo_app.c amalgam/jerryscript.c amalgam/jerryscript-port-default.c amalgam/jerryscript-math.c -Iamalgam/ ``` -Please note that the headers must be available on the include path. +(Note: The headers must be available on the include path.) -In addition there is a `-DENABLE_ALL_IN_ONE_SOURCE=ON` CMake option to use this kind of sources during the build. +This approach is also under the hood of the [amalgamated build](#amalgamated-build) process. # Target specific information diff --git a/jerry-core/CMakeLists.txt b/jerry-core/CMakeLists.txt index 1fa9e80b0..bdb20c6c6 100644 --- a/jerry-core/CMakeLists.txt +++ b/jerry-core/CMakeLists.txt @@ -18,9 +18,6 @@ project (${JERRY_CORE_NAME} C) include(CheckLibraryExists) -# Optional build settings -set(ENABLE_ALL_IN_ONE OFF CACHE BOOL "Enable all-in-one build?") - # Optional features set(JERRY_CPOINTER_32_BIT OFF CACHE BOOL "Enable 32 bit compressed pointers?") set(JERRY_DEBUGGER OFF CACHE BOOL "Enable JerryScript debugger?") @@ -46,12 +43,6 @@ set(JERRY_STACK_LIMIT "(0)" CACHE STRING "Maximum stack usa set(JERRY_GC_MARK_LIMIT "(8)" CACHE STRING "Maximum depth of recursion during GC mark phase") # Option overrides -if(USING_MSVC) - set(ENABLE_ALL_IN_ONE ON) # FIXME: This should not be needed but right now it is. To be tracked down and followed up. - - set(ENABLE_ALL_IN_ONE_MESSAGE " (FORCED BY COMPILER)") -endif() - if(JERRY_SYSTEM_ALLOCATOR) set(JERRY_CPOINTER_32_BIT ON) @@ -85,7 +76,6 @@ if(JERRY_MEM_STATS OR JERRY_PARSER_DUMP_BYTE_CODE OR JERRY_REGEXP_DUMP_BYTE_CODE endif() # Status messages -message(STATUS "ENABLE_ALL_IN_ONE " ${ENABLE_ALL_IN_ONE} ${ENABLE_ALL_IN_ONE_MESSAGE}) message(STATUS "JERRY_CPOINTER_32_BIT " ${JERRY_CPOINTER_32_BIT} ${JERRY_CPOINTER_32_BIT_MESSAGE}) message(STATUS "JERRY_DEBUGGER " ${JERRY_DEBUGGER}) message(STATUS "JERRY_ERROR_MESSAGES " ${JERRY_ERROR_MESSAGES}) @@ -147,55 +137,41 @@ file(GLOB SOURCE_CORE_FILES parser/regexp/*.c vm/*.c) -# All-in-one build -if(ENABLE_ALL_IN_ONE) - set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/jerry-all-in.c") - list(SORT SOURCE_CORE_FILES) - file(REMOVE ${ALL_IN_FILE}) - - foreach(FILE ${SOURCE_CORE_FILES}) - file(APPEND ${ALL_IN_FILE} "#include \"${FILE}\"\n") - endforeach() - - set(SOURCE_CORE_FILES ${ALL_IN_FILE}) -endif() - -# "Single" JerryScript source/header build. +# Amalgamated JerryScript source/header build. # The process will create the following files: # * jerryscript.c # * jerryscript.h # * jerryscript-config.h -if(ENABLE_ALL_IN_ONE_SOURCE) +if(ENABLE_AMALGAM) # Create single C/H file file(GLOB HEADER_CORE_FILES *.h) # Generated files - set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript.c") - set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/jerryscript.h") - set(JERRYSCRIPT_CONFIG_H "${CMAKE_BINARY_DIR}/src/jerryscript-config.h") + set(AMALGAM_CORE_C "${CMAKE_BINARY_DIR}/amalgam/jerryscript.c") + set(AMALGAM_CORE_H "${CMAKE_BINARY_DIR}/amalgam/jerryscript.h") + set(AMALGAM_CONFIG_H "${CMAKE_BINARY_DIR}/amalgam/jerryscript-config.h") - add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} - COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py + add_custom_command(OUTPUT ${AMALGAM_CORE_C} ${AMALGAM_CORE_H} + COMMAND python ${CMAKE_SOURCE_DIR}/tools/amalgam.py --jerry-core - --output-dir ${CMAKE_BINARY_DIR}/src + --output-dir ${CMAKE_BINARY_DIR}/amalgam DEPENDS ${SOURCE_CORE_FILES} ${HEADER_CORE_FILES} - ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py - ${CMAKE_SOURCE_DIR}/tools/srcmerger.py + ${CMAKE_SOURCE_DIR}/tools/amalgam.py ) # The "true" jerryscript-config.h will be generated by the configure_file below, # which contains the default options and the ones passed for the CMake. # The input for this is the jerryscript-config.h generated by the command above. set(JERRYSCRIPT_GEN_CONFIG_H ${CMAKE_CURRENT_BINARY_DIR}/jerryscript-config.h) - add_custom_command(OUTPUT ${JERRYSCRIPT_CONFIG_H} - COMMAND ${CMAKE_COMMAND} -E copy ${JERRYSCRIPT_GEN_CONFIG_H} ${JERRYSCRIPT_CONFIG_H} - DEPENDS ${ALL_IN_FILE_C} ${ALL_IN_FILE_H}) - add_custom_target(generate-single-source-jerry DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H}) - add_dependencies(generate-single-source generate-single-source-jerry) + add_custom_command(OUTPUT ${AMALGAM_CONFIG_H} + COMMAND ${CMAKE_COMMAND} -E copy ${JERRYSCRIPT_GEN_CONFIG_H} ${AMALGAM_CONFIG_H} + DEPENDS ${AMALGAM_CORE_C} ${AMALGAM_CORE_H}) + add_custom_target(amalgam-jerry DEPENDS ${AMALGAM_CORE_C} ${AMALGAM_CORE_H} ${AMALGAM_CONFIG_H}) + add_dependencies(amalgam amalgam-jerry) - set(SOURCE_CORE_FILES ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}) + set(SOURCE_CORE_FILES ${AMALGAM_CORE_C} ${AMALGAM_CORE_H} ${AMALGAM_CONFIG_H}) endif() # Third-party diff --git a/jerry-math/CMakeLists.txt b/jerry-math/CMakeLists.txt index 5df553074..e7ecc6f84 100644 --- a/jerry-math/CMakeLists.txt +++ b/jerry-math/CMakeLists.txt @@ -30,28 +30,27 @@ set(INCLUDE_MATH "${CMAKE_CURRENT_SOURCE_DIR}/include") # Source directories file(GLOB SOURCE_MATH *.c) -# "Single" JerryScript libm source/header build. +# Amalgamated JerryScript source/header build. # The process will create the following files: # * jerryscript-math.c # * math.h -if(ENABLE_ALL_IN_ONE_SOURCE) +if(ENABLE_AMALGAM) file(GLOB HEADER_MATH *.h) - set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript-math.c") - set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/math.h") + set(AMALGAM_MATH_C "${CMAKE_BINARY_DIR}/amalgam/jerryscript-math.c") + set(AMALGAM_MATH_H "${CMAKE_BINARY_DIR}/amalgam/math.h") - add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H} - COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py + add_custom_command(OUTPUT ${AMALGAM_MATH_C} ${AMALGAM_MATH_H} + COMMAND python ${CMAKE_SOURCE_DIR}/tools/amalgam.py --jerry-math - --output-dir ${CMAKE_BINARY_DIR}/src + --output-dir ${CMAKE_BINARY_DIR}/amalgam DEPENDS ${SOURCE_MATH} ${HEADER_MATH} - ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py - ${CMAKE_SOURCE_DIR}/tools/srcmerger.py + ${CMAKE_SOURCE_DIR}/tools/amalgam.py ) - add_custom_target(generate-single-source-math DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H}) - add_dependencies(generate-single-source generate-single-source-math) + add_custom_target(amalgam-math DEPENDS ${AMALGAM_MATH_C} ${AMALGAM_MATH_H}) + add_dependencies(amalgam amalgam-math) - set(SOURCE_MATH ${ALL_IN_FILE} ${ALL_IN_FILE_H}) + set(SOURCE_MATH ${AMALGAM_MATH_C} ${AMALGAM_MATH_H}) endif() add_library(${JERRY_MATH_NAME} ${SOURCE_MATH}) diff --git a/jerry-port/default/CMakeLists.txt b/jerry-port/default/CMakeLists.txt index e9ac6754b..2bf44ea6d 100644 --- a/jerry-port/default/CMakeLists.txt +++ b/jerry-port/default/CMakeLists.txt @@ -22,28 +22,27 @@ set(INCLUDE_PORT_DEFAULT "${CMAKE_CURRENT_SOURCE_DIR}/include") # Source directories file(GLOB SOURCE_PORT_DEFAULT *.c) -# "Single" JerryScript source/header build. +# Amalgamated JerryScript source/header build. # The process will create the following files: # * jerryscript-port-default.c # * jerryscript-port-default.h -if(ENABLE_ALL_IN_ONE_SOURCE) +if(ENABLE_AMALGAM) file(GLOB HEADER_PORT_DEFAULT *.h) - set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript-port-default.c") - set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/jerryscript-port-default.h") + set(AMALGAM_PORT_C "${CMAKE_BINARY_DIR}/amalgam/jerryscript-port-default.c") + set(AMALGAM_PORT_H "${CMAKE_BINARY_DIR}/amalgam/jerryscript-port-default.h") - add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H} - COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py + add_custom_command(OUTPUT ${AMALGAM_PORT_C} ${AMALGAM_PORT_H} + COMMAND python ${CMAKE_SOURCE_DIR}/tools/amalgam.py --jerry-port-default - --output-dir ${CMAKE_BINARY_DIR}/src + --output-dir ${CMAKE_BINARY_DIR}/amalgam DEPENDS ${SOURCE_PORT_DEFAULT} ${HEADER_PORT_DEFAULT} - ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py - ${CMAKE_SOURCE_DIR}/tools/srcmerger.py + ${CMAKE_SOURCE_DIR}/tools/amalgam.py ) - add_custom_target(generate-single-source-port DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H}) - add_dependencies(generate-single-source generate-single-source-port) + add_custom_target(amalgam-port DEPENDS ${AMALGAM_PORT_C} ${AMALGAM_PORT_H}) + add_dependencies(amalgam amalgam-port) - set(SOURCE_PORT_DEFAULT ${ALL_IN_FILE} ${ALL_IN_FILE_H}) + set(SOURCE_PORT_DEFAULT ${AMALGAM_PORT_C} ${AMALGAM_PORT_H}) endif() # Define _BSD_SOURCE and _DEFAULT_SOURCE diff --git a/targets/esp8266/Makefile.esp8266 b/targets/esp8266/Makefile.esp8266 index ddc60c149..9c131a93c 100644 --- a/targets/esp8266/Makefile.esp8266 +++ b/targets/esp8266/Makefile.esp8266 @@ -49,7 +49,7 @@ jerry: -DCMAKE_C_COMPILER=xtensa-lx106-elf-gcc \ -DCMAKE_C_COMPILER_WORKS=TRUE \ -DENABLE_LTO=OFF \ - -DENABLE_ALL_IN_ONE=ON \ + -DENABLE_AMALGAM=ON \ -DJERRY_MATH=ON \ -DJERRY_CMDLINE=OFF \ -DJERRY_PROFILE="es5.1" \ diff --git a/targets/nuttx-stm32f4/Makefile.travis b/targets/nuttx-stm32f4/Makefile.travis index 8da37f7e9..50eadc14d 100644 --- a/targets/nuttx-stm32f4/Makefile.travis +++ b/targets/nuttx-stm32f4/Makefile.travis @@ -51,7 +51,7 @@ install: install-apt-get-deps install-noapt # Build JerryScript. script-build-jerryscript: - tools/build.py --clean --toolchain cmake/toolchain_mcu_stm32f4.cmake --profile=es.next --jerry-cmdline OFF --lto OFF --jerry-math ON --all-in-one ON --jerry-port-default OFF --mem-heap 70 --compile-flag='--sysroot=../nuttx' + tools/build.py --clean --toolchain cmake/toolchain_mcu_stm32f4.cmake --profile=es.next --jerry-cmdline OFF --lto OFF --jerry-math ON --amalgam ON --jerry-port-default OFF --mem-heap 70 --compile-flag='--sysroot=../nuttx' # Link in the NuttX JerryScript target directory under the NuttX apps tree. script-add-jerryscript-app: diff --git a/targets/nuttx-stm32f4/README.md b/targets/nuttx-stm32f4/README.md index c1850c830..2eba1d766 100644 --- a/targets/nuttx-stm32f4/README.md +++ b/targets/nuttx-stm32f4/README.md @@ -42,7 +42,7 @@ jerryscript/tools/build.py \ --lto=OFF \ --jerry-cmdline=OFF \ --jerry-math=ON \ - --all-in-one=ON \ + --amalgam=ON \ --mem-heap=70 \ --profile=es.next \ --compile-flag="--sysroot=${PWD}/nuttx" \ diff --git a/targets/particle/Makefile.particle b/targets/particle/Makefile.particle index b848315f7..1eb87a31f 100644 --- a/targets/particle/Makefile.particle +++ b/targets/particle/Makefile.particle @@ -44,7 +44,6 @@ jerrycore: -DCMAKE_C_COMPILER=arm-none-eabi-gcc \ -DCMAKE_C_COMPILER_WORKS=TRUE \ -DENABLE_LTO=ON \ - -DENABLE_ALL_IN_ONE=OFF \ -DJERRY_CMDLINE=OFF \ -DJERRY_PROFILE=minimal \ -DENABLE_STRIP=OFF \ diff --git a/targets/riot-stm32f4/Makefile.riot b/targets/riot-stm32f4/Makefile.riot index 2187c250a..1600ba75c 100644 --- a/targets/riot-stm32f4/Makefile.riot +++ b/targets/riot-stm32f4/Makefile.riot @@ -46,7 +46,6 @@ libjerry: -DCMAKE_C_COMPILER=$(CC) \ -DCMAKE_C_COMPILER_WORKS=TRUE \ -DENABLE_LTO=OFF \ - -DENABLE_ALL_IN_ONE=OFF \ -DJERRY_CMDLINE=OFF \ -DJERRY_PROFILE="es5.1" \ -DEXTERNAL_COMPILE_FLAGS="$(EXT_CFLAGS)" \ diff --git a/targets/zephyr/Makefile.zephyr b/targets/zephyr/Makefile.zephyr index 583e37c0b..d182df34d 100644 --- a/targets/zephyr/Makefile.zephyr +++ b/targets/zephyr/Makefile.zephyr @@ -82,7 +82,6 @@ endif -DCMAKE_C_COMPILER="$(subst ccache ,,$(CC))" \ -DCMAKE_C_COMPILER_WORKS=TRUE \ -DENABLE_LTO=OFF \ - -DENABLE_ALL_IN_ONE=OFF \ -DJERRY_CMDLINE=OFF \ -DJERRY_PROFILE=$(JERRYPROFILE) \ -DJERRY_ERROR_MESSAGES=ON \ diff --git a/tools/srcmerger.py b/tools/amalgam.py old mode 100644 new mode 100755 similarity index 53% rename from tools/srcmerger.py rename to tools/amalgam.py index 48dd4745f..4d7888736 --- a/tools/srcmerger.py +++ b/tools/amalgam.py @@ -13,30 +13,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from __future__ import print_function import argparse import fnmatch +import json import logging import os import re -import sys +import shutil + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +JERRY_CORE = os.path.join(ROOT_DIR, 'jerry-core') +JERRY_PORT = os.path.join(ROOT_DIR, 'jerry-port', 'default') +JERRY_MATH = os.path.join(ROOT_DIR, 'jerry-math') -class SourceMerger(object): +class Amalgamator(object): # pylint: disable=too-many-instance-attributes _RE_INCLUDE = re.compile(r'\s*#include ("|<)(.*?)("|>)\n$') - def __init__(self, h_files, extra_includes=None, remove_includes=None, add_lineinfo=False): - self._log = logging.getLogger('sourcemerger') + def __init__(self, h_files, extra_includes=(), remove_includes=(), add_lineinfo=False): + self._h_files = h_files + self._extra_includes = extra_includes + self._remove_includes = remove_includes + self._add_lineinfo = add_lineinfo self._last_builtin = None self._processed = [] self._output = [] - self._h_files = h_files - self._extra_includes = extra_includes or [] - self._remove_includes = remove_includes - self._add_lineinfo = add_lineinfo # The copyright will be loaded from the first input file self._copyright = {'lines': [], 'loaded': False} @@ -44,8 +50,8 @@ class SourceMerger(object): # Special case #2: Builtin include header name usage if line.strip() == "#include BUILTIN_INC_HEADER_NAME": assert self._last_builtin is not None, 'No previous BUILTIN_INC_HEADER_NAME definition' - self._log.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: %s', - file_level, self._last_builtin) + logging.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: "%s"', + file_level, self._last_builtin) self.add_file(self._h_files[self._last_builtin]) # return from the function as we have processed the included file return @@ -54,8 +60,7 @@ class SourceMerger(object): if line.startswith('#define BUILTIN_INC_HEADER_NAME '): # the line is in this format: #define BUILTIN_INC_HEADER_NAME "" self._last_builtin = line.split('"', 2)[1] - self._log.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: %s', - file_level, self._last_builtin) + logging.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: "%s"', file_level, self._last_builtin) # the line is not anything special, just push it into the output self._output.append(line) @@ -75,11 +80,11 @@ class SourceMerger(object): def add_file(self, filename, file_level=0): if os.path.basename(filename) in self._processed: - self._log.warning('Tried to to process an already processed file: "%s"', filename) + logging.warning('Tried to to process an already processed file: "%s"', filename) return if not file_level: - self._log.debug('Adding file: "%s"', filename) + logging.debug('Adding file: "%s"', filename) file_level += 1 @@ -111,7 +116,7 @@ class SourceMerger(object): continue # check if the line is an '#include' line - match = SourceMerger._RE_INCLUDE.match(line) + match = self._RE_INCLUDE.match(line) if not match: # the line is not a header self._process_non_include(line, file_level) @@ -125,27 +130,24 @@ class SourceMerger(object): name = match.group(2) if name in self._remove_includes: - self._log.debug('[%d] Removing include line (%s:%d): %s', - file_level, filename, line_idx, line.strip()) + logging.debug('[%d] Removing include line (%s:%d): %s', + file_level, filename, line_idx, line.strip()) # emit a line info so the line numbering can be tracked correctly self._emit_lineinfo(line_idx + 1, filename) continue if name not in self._h_files: - self._log.warning('[%d] Include not found: "%s" in "%s:%d"', - file_level, name, filename, line_idx) + logging.warning('[%d] Include not found (%s:%d): "%s"', file_level, filename, line_idx, name) self._output.append(line) continue if name in self._processed: - self._log.debug('[%d] Already included: "%s"', - file_level, name) + logging.debug('[%d] Already included: "%s"', file_level, name) # emit a line info so the line numbering can be tracked correctly self._emit_lineinfo(line_idx + 1, filename) continue - self._log.debug('[%d] Including: "%s"', - file_level, self._h_files[name]) + logging.debug('[%d] Including: "%s"', file_level, self._h_files[name]) self.add_file(self._h_files[name], file_level) # mark the continuation of the current file in the output @@ -198,7 +200,7 @@ def collect_files(base_dir, pattern): name = os.path.basename(fname) if name in name_mapping: - print('Duplicate name detected: "%s" and "%s"' % (name, name_mapping[name])) + logging.warning('Duplicate name detected: "%s" and "%s"', fname, name_mapping[name]) continue name_mapping[name] = fname @@ -206,61 +208,154 @@ def collect_files(base_dir, pattern): return name_mapping -def run_merger(args): - h_files = collect_files(args.base_dir, '*.h') - c_files = collect_files(args.base_dir, '*.c') +def amalgamate(base_dir, input_files=(), output_file=None, + append_c_files=False, remove_includes=(), extra_includes=(), + add_lineinfo=False): + """ + :param input_files: Main input source/header files + :param output_file: Output source/header file + :param append_c_files: Enable auto inclusion of c files under the base-dir + :param add_lineinfo: Enable #line macro insertion into the generated sources + """ + logging.debug('Starting merge with args: %s', json.dumps(locals(), indent=4, sort_keys=True)) - for name in args.remove_include: + h_files = collect_files(base_dir, '*.h') + c_files = collect_files(base_dir, '*.c') + + for name in remove_includes: c_files.pop(name, '') h_files.pop(name, '') - merger = SourceMerger(h_files, args.push_include, args.remove_include, args.add_lineinfo) - for input_file in args.input_files: - merger.add_file(input_file) + amalgam = Amalgamator(h_files, extra_includes, remove_includes, add_lineinfo) + for input_file in input_files: + amalgam.add_file(input_file) - if args.append_c_files: + if append_c_files: # if the input file is in the C files list it should be removed to avoid # double inclusion of the file - for input_file in args.input_files: + for input_file in input_files: input_name = os.path.basename(input_file) c_files.pop(input_name, '') # Add the C files in reverse order to make sure that builtins are # not at the beginning. for fname in sorted(c_files.values(), reverse=True): - merger.add_file(fname) + amalgam.add_file(fname) - with open(args.output_file, 'w') as output: - merger.write_output(output) + with open(output_file, 'w') as output: + amalgam.write_output(output) +def amalgamate_jerry_core(output_dir): + amalgamate( + base_dir=JERRY_CORE, + input_files=[ + os.path.join(JERRY_CORE, 'api', 'jerry.c'), + # Add the global built-in by default to include some common items + # to avoid problems with common built-in headers + os.path.join(JERRY_CORE, 'ecma', 'builtin-objects', 'ecma-builtins.c'), + ], + output_file=os.path.join(output_dir, 'jerryscript.c'), + append_c_files=True, + remove_includes=[ + 'jerryscript.h', + 'jerryscript-port.h', + 'jerryscript-compiler.h', + 'jerryscript-core.h', + 'jerryscript-debugger.h', + 'jerryscript-debugger-transport.h', + 'jerryscript-port.h', + 'jerryscript-snapshot.h', + 'config.h', + ], + extra_includes=['jerryscript.h'], + ) + + amalgamate( + base_dir=JERRY_CORE, + input_files=[ + os.path.join(JERRY_CORE, 'include', 'jerryscript.h'), + os.path.join(JERRY_CORE, 'include', 'jerryscript-debugger-transport.h'), + ], + output_file=os.path.join(output_dir, 'jerryscript.h'), + remove_includes=['config.h'], + extra_includes=['jerryscript-config.h'], + ) + + shutil.copyfile(os.path.join(JERRY_CORE, 'config.h'), + os.path.join(output_dir, 'jerryscript-config.h')) + + +def amalgamate_jerry_port_default(output_dir): + amalgamate( + base_dir=JERRY_PORT, + output_file=os.path.join(output_dir, 'jerryscript-port-default.c'), + append_c_files=True, + remove_includes=[ + 'jerryscript-port.h', + 'jerryscript-port-default.h', + 'jerryscript-debugger.h', + ], + extra_includes=[ + 'jerryscript.h', + 'jerryscript-port-default.h', + ], + ) + + amalgamate( + base_dir=JERRY_PORT, + input_files=[os.path.join(JERRY_PORT, 'include', 'jerryscript-port-default.h')], + output_file=os.path.join(output_dir, 'jerryscript-port-default.h'), + remove_includes=[ + 'jerryscript-port.h', + 'jerryscript.h', + ], + extra_includes=['jerryscript.h'], + ) + + +def amalgamate_jerry_math(output_dir): + amalgamate( + base_dir=JERRY_MATH, + output_file=os.path.join(output_dir, 'jerryscript-math.c'), + append_c_files=True, + ) + + shutil.copyfile(os.path.join(JERRY_MATH, 'include', 'math.h'), + os.path.join(output_dir, 'math.h')) + def main(): - parser = argparse.ArgumentParser(description='Merge source/header files.') - parser.add_argument('--base-dir', metavar='DIR', type=str, dest='base_dir', - help='', default=os.path.curdir) - parser.add_argument('--input', metavar='FILES', type=str, action='append', dest='input_files', - help='Main input source/header files', default=[]) - parser.add_argument('--output', metavar='FILE', type=str, dest='output_file', - help='Output source/header file') - parser.add_argument('--append-c-files', dest='append_c_files', default=False, - action='store_true', help='Enable auto inclusion of c files under the base-dir') - parser.add_argument('--remove-include', action='append', default=[]) - parser.add_argument('--push-include', action='append', default=[]) - parser.add_argument('--add-lineinfo', action='store_true', default=False, - help='Enable #line macro insertion into the generated sources') - parser.add_argument('--verbose', '-v', action='store_true', default=False) + parser = argparse.ArgumentParser(description='Generate amalgamated sources.') + parser.add_argument('--jerry-core', action='store_true', + help='amalgamate jerry-core files') + parser.add_argument('--jerry-port-default', action='store_true', + help='amalgamate jerry-port-default files') + parser.add_argument('--jerry-math', action='store_true', + help='amalgamate jerry-math files') + parser.add_argument('--output-dir', metavar='DIR', default='amalgam', + help='output dir (default: %(default)s)') + parser.add_argument('--verbose', '-v', action='store_true', + help='increase verbosity') args = parser.parse_args() - log_level = logging.WARNING if args.verbose: - log_level = logging.DEBUG + logging.basicConfig(level=logging.DEBUG) - logging.basicConfig(level=log_level) - logging.debug('Starting merge with args: %s', str(sys.argv)) + try: + os.makedirs(args.output_dir) + except os.error: + pass - run_merger(args) + if args.jerry_core: + amalgamate_jerry_core(args.output_dir) + + if args.jerry_port_default: + amalgamate_jerry_port_default(args.output_dir) + + if args.jerry_math: + amalgamate_jerry_math(args.output_dir) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/tools/build.py b/tools/build.py index fa310c59b..9b140c764 100755 --- a/tools/build.py +++ b/tools/build.py @@ -76,6 +76,8 @@ def get_arguments(): help='add custom library to be linked') buildgrp.add_argument('--linker-flag', metavar='OPT', action='append', default=[], help='add custom linker flag') + buildgrp.add_argument('--amalgam', metavar='X', choices=['ON', 'OFF'], type=str.upper, + help='enable amalgamated build (%(choices)s)') buildgrp.add_argument('--lto', metavar='X', choices=['ON', 'OFF'], type=str.upper, help='enable link-time optimizations (%(choices)s)') buildgrp.add_argument('--shared-libs', metavar='X', choices=['ON', 'OFF'], type=str.upper, @@ -108,8 +110,6 @@ def get_arguments(): help=devhelp('build unittests (%(choices)s)')) coregrp = parser.add_argument_group('jerry-core options') - coregrp.add_argument('--all-in-one', metavar='X', choices=['ON', 'OFF'], type=str.upper, - help='all-in-one build (%(choices)s)') coregrp.add_argument('--cpointer-32bit', metavar='X', choices=['ON', 'OFF'], type=str.upper, help='enable 32 bit compressed pointers (%(choices)s)') coregrp.add_argument('--error-messages', metavar='X', choices=['ON', 'OFF'], type=str.upper, @@ -180,6 +180,7 @@ def generate_build_options(arguments): build_options_append('EXTERNAL_COMPILE_FLAGS', ' '.join(arguments.compile_flag)) build_options_append('EXTERNAL_LINK_LIBS', ' '.join(arguments.link_lib)) build_options_append('EXTERNAL_LINKER_FLAGS', ' '.join(arguments.linker_flag)) + build_options_append('ENABLE_AMALGAM', arguments.amalgam) build_options_append('ENABLE_LTO', arguments.lto) build_options_append('BUILD_SHARED_LIBS', arguments.shared_libs) build_options_append('ENABLE_STRIP', arguments.strip) @@ -198,7 +199,6 @@ def generate_build_options(arguments): build_options_append('UNITTESTS', arguments.unittests) # jerry-core options - build_options_append('ENABLE_ALL_IN_ONE', arguments.all_in_one) build_options_append('JERRY_CPOINTER_32_BIT', arguments.cpointer_32bit) build_options_append('JERRY_ERROR_MESSAGES', arguments.error_messages) build_options_append('JERRY_EXTERNAL_CONTEXT', arguments.external_context) diff --git a/tools/run-tests.py b/tools/run-tests.py index c544c137f..1d664906d 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -129,8 +129,8 @@ JERRY_BUILDOPTIONS = [ ['--error-messages=on']), Options('buildoption_test-logging', ['--logging=on']), - Options('buildoption_test-all_in_one', - ['--all-in-one=on']), + Options('buildoption_test-amalgam', + ['--amalgam=on']), Options('buildoption_test-valgrind', ['--valgrind=on']), Options('buildoption_test-mem_stats', @@ -163,8 +163,6 @@ JERRY_BUILDOPTIONS = [ OPTIONS_STACK_LIMIT), Options('buildoption_test-gc-mark_limit', OPTIONS_GC_MARK_LIMIT), - Options('buildoption_test-single-source', - ['--cmake-param=-DENABLE_ALL_IN_ONE_SOURCE=ON']), Options('buildoption_test-jerry-debugger', ['--jerry-debugger=on']), Options('buildoption_test-module-off', diff --git a/tools/srcgenerator.py b/tools/srcgenerator.py deleted file mode 100644 index 9d572909f..000000000 --- a/tools/srcgenerator.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python - -# Copyright JS Foundation and other contributors, http://js.foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function - -import argparse -import logging -import os -import subprocess -import shutil - - -TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) -ROOT_DIR = os.path.dirname(TOOLS_DIR) -SRCMERGER = os.path.join(TOOLS_DIR, 'srcmerger.py') -JERRY_CORE = os.path.join(ROOT_DIR, 'jerry-core') -JERRY_PORT = os.path.join(ROOT_DIR, 'jerry-port', 'default') -JERRY_MATH = os.path.join(ROOT_DIR, 'jerry-math') - - -def run_commands(*cmds, **kwargs): - log = logging.getLogger('sourcegenerator') - verbose = kwargs.get('verbose', False) - - for cmd in cmds: - if verbose: - cmd.append('--verbose') - log.debug('Run command: %s', cmd) - subprocess.call(cmd) - - -def generate_jerry_core(output_dir, verbose=False): - cmd_jerry_c_gen = [ - 'python', SRCMERGER, - '--base-dir', JERRY_CORE, - '--input={}/api/jerry.c'.format(JERRY_CORE), - '--output={}/jerryscript.c'.format(output_dir), - '--append-c-files', - # Add the global built-in by default to include some common items - # to avoid problems with common built-in headers - '--input={}/ecma/builtin-objects/ecma-builtins.c'.format(JERRY_CORE), - '--remove-include=jerryscript.h', - '--remove-include=jerryscript-port.h', - '--remove-include=jerryscript-compiler.h', - '--remove-include=jerryscript-core.h', - '--remove-include=jerryscript-debugger.h', - '--remove-include=jerryscript-debugger-transport.h', - '--remove-include=jerryscript-port.h', - '--remove-include=jerryscript-snapshot.h', - '--remove-include=config.h', - '--push-include=jerryscript.h', - ] - - cmd_jerry_h_gen = [ - 'python', SRCMERGER, - '--base-dir', JERRY_CORE, - '--input={}/include/jerryscript.h'.format(JERRY_CORE), - '--input={}/include/jerryscript-debugger-transport.h'.format(JERRY_CORE), - '--output={}/jerryscript.h'.format(output_dir), - '--remove-include=config.h', - '--push-include=jerryscript-config.h', - ] - - run_commands(cmd_jerry_c_gen, cmd_jerry_h_gen, verbose=verbose) - - shutil.copyfile('{}/config.h'.format(JERRY_CORE), - '{}/jerryscript-config.h'.format(output_dir)) - - -def generate_jerry_port_default(output_dir, verbose=False): - cmd_port_c_gen = [ - 'python', SRCMERGER, - '--base-dir', JERRY_PORT, - '--output={}/jerryscript-port-default.c'.format(output_dir), - '--append-c-files', - '--remove-include=jerryscript-port.h', - '--remove-include=jerryscript-port-default.h', - '--remove-include=jerryscript-debugger.h', - '--push-include=jerryscript.h', - '--push-include=jerryscript-port-default.h', - ] - - cmd_port_h_gen = [ - 'python', SRCMERGER, - '--base-dir', JERRY_PORT, - '--input={}/include/jerryscript-port-default.h'.format(JERRY_PORT), - '--output={}/jerryscript-port-default.h'.format(output_dir), - '--remove-include=jerryscript-port.h', - '--remove-include=jerryscript.h', - '--push-include=jerryscript.h', - ] - - run_commands(cmd_port_c_gen, cmd_port_h_gen, verbose=verbose) - - -def generate_jerry_math(output_dir, verbose=False): - cmd_math_c_gen = [ - 'python', SRCMERGER, - '--base-dir', JERRY_MATH, - '--output={}/jerryscript-math.c'.format(output_dir), - '--append-c-files', - ] - - run_commands(cmd_math_c_gen, verbose=verbose) - - shutil.copyfile('{}/include/math.h'.format(JERRY_MATH), - '{}/math.h'.format(output_dir)) - -def main(): - parser = argparse.ArgumentParser(description='Generate single sources.') - parser.add_argument('--jerry-core', action='store_true', dest='jerry_core', - help='Generate jerry-core files', default=False) - parser.add_argument('--jerry-port-default', action='store_true', dest='jerry_port_default', - help='Generate jerry-port-default files', default=False) - parser.add_argument('--jerry-math', action='store_true', dest='jerry_math', - help='Generate jerry-math files', default=False) - parser.add_argument('--output-dir', metavar='DIR', type=str, dest='output_dir', - default='gen_src', help='Output dir') - parser.add_argument('--verbose', '-v', action='store_true', default=False) - - args = parser.parse_args() - - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - - try: - os.makedirs(args.output_dir) - except os.error: - pass - - if args.jerry_core: - generate_jerry_core(args.output_dir, args.verbose) - - if args.jerry_port_default: - generate_jerry_port_default(args.output_dir, args.verbose) - - if args.jerry_math: - generate_jerry_math(args.output_dir, args.verbose) - - -if __name__ == '__main__': - main()