From 3b28178deb7a42a2df8c19b9bf190bb5da732b33 Mon Sep 17 00:00:00 2001 From: ruro Date: Wed, 18 Aug 2021 01:46:59 +0300 Subject: [PATCH] `--parallel` improvements (#13800) * improve make parallel jobs support * document the --parallel option * disable the output-sync for interactive targets --- docs/cli_commands.md | 17 ++++++++++++++--- lib/python/qmk/cli/compile.py | 2 +- lib/python/qmk/cli/flash.py | 2 +- lib/python/qmk/cli/multibuild.py | 6 +++--- lib/python/qmk/commands.py | 23 ++++++++++++++++++++--- message.mk | 4 ++++ platforms/arm_atsam/flash.mk | 2 +- platforms/avr/flash.mk | 14 +++++++------- platforms/chibios/flash.mk | 10 +++++----- 9 files changed, 56 insertions(+), 24 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index c7468eb5e71..8fa7ad41dc8 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -17,7 +17,7 @@ qmk compile [-c] **Usage for Keymaps**: ``` -qmk compile [-c] [-e =] -kb -km +qmk compile [-c] [-e =] [-j ] -kb -km ``` **Usage in Keyboard Directory**: @@ -73,6 +73,17 @@ $ qmk compile -kb dz60 ... ``` +**Parallel Compilation**: + +It is possible to speed up compilation by adding the `-j`/`--parallel` flag. +``` +qmk compile -j -kb +``` +The `num_jobs` argument determines the maximum number of jobs that can be used. Setting it to zero will enable parallel compilation without limiting the maximum number of jobs. +``` +qmk compile -j 0 -kb +``` + ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl `. Visit the [Flashing Firmware](flashing.md) guide for more details of the available bootloaders. @@ -82,13 +93,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K **Usage for Configurator Exports**: ``` -qmk flash [-bl ] [-c] [-e =] +qmk flash [-bl ] [-c] [-e =] [-j ] ``` **Usage for Keymaps**: ``` -qmk flash -kb -km [-bl ] [-c] [-e =] +qmk flash -kb -km [-bl ] [-c] [-e =] [-j ] ``` **Listing the Bootloaders** diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7a45e77214b..acbd7786498 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,7 @@ from qmk.keymap import keymap_completer @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 1b2932a5b27..c2d9e09c693 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -38,7 +38,7 @@ def print_bootloader_help(): @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('QMK Flash.') diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py index bdb0b493c80..85ed0fa1e98 100755 --- a/lib/python/qmk/cli/multibuild.py +++ b/lib/python/qmk/cli/multibuild.py @@ -10,7 +10,7 @@ from subprocess import DEVNULL from milc import cli from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make +from qmk.commands import _find_make, get_make_parallel_args import qmk.keyboard import qmk.keymap @@ -28,7 +28,7 @@ def _is_split(keyboard_name): return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") @@ -80,7 +80,7 @@ all: {keyboard_safe}_binary ) # yapf: enable - cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) + cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) # Check for failures failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 8c66228b2b7..421453d8377 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -52,7 +52,7 @@ def create_make_target(target, parallel=1, **env_vars): for key, value in env_vars.items(): env.append(f'{key}={value}') - return [make_cmd, '-j', str(parallel), *env, target] + return [make_cmd, *get_make_parallel_args(parallel), *env, target] def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): @@ -112,6 +112,24 @@ def get_git_version(current_time, repo_dir='.', check_dir='.'): return current_time +def get_make_parallel_args(parallel=1): + """Returns the arguments for running the specified number of parallel jobs. + """ + parallel_args = [] + + if int(parallel) <= 0: + # 0 or -1 means -j without argument (unlimited jobs) + parallel_args.append('--jobs') + else: + parallel_args.append('--jobs=' + str(parallel)) + + if int(parallel) != 1: + # If more than 1 job is used, synchronize parallel output by target + parallel_args.append('--output-sync=target') + + return parallel_args + + def create_version_h(skip_git=False, skip_all=False): """Generate version.h contents """ @@ -185,8 +203,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va make_command.append('-s') make_command.extend([ - '-j', - str(parallel), + *get_make_parallel_args(parallel), '-r', '-R', '-f', diff --git a/message.mk b/message.mk index dcd47e55019..419e3f5f314 100644 --- a/message.mk +++ b/message.mk @@ -36,6 +36,10 @@ PRINT_OK = $(SILENT) || printf " $(OK_STRING)" | $(AWK_STATUS) BUILD_CMD = LOG=$$($(CMD) 2>&1) ; if [ $$? -gt 0 ]; then $(PRINT_ERROR); elif [ "$$LOG" != "" ] ; then $(PRINT_WARNING); else $(PRINT_OK); fi; MAKE_MSG_FORMAT = $(AWK) '{ printf "%-118s", $$0;}' +# The UNSYNC_OUTPUT_CMD command disables the `--output-sync` for the current command, if the `--output-sync` granularity is `target` or lower. +# This is achieved by telling make to treat the current command as if it invokes a recursive make subcommand (as if by calling `$(MAKE)`). +UNSYNC_OUTPUT_CMD = +true + # Define Messages # English MSG_ERRORS_NONE = Errors: none diff --git a/platforms/arm_atsam/flash.mk b/platforms/arm_atsam/flash.mk index 444ba77f53f..f31d4b4d95a 100644 --- a/platforms/arm_atsam/flash.mk +++ b/platforms/arm_atsam/flash.mk @@ -5,7 +5,7 @@ flash: bin ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_ARCH)" endif diff --git a/platforms/avr/flash.mk b/platforms/avr/flash.mk index e679bf0ace7..8eef5fbb03e 100644 --- a/platforms/avr/flash.mk +++ b/platforms/avr/flash.mk @@ -161,19 +161,19 @@ hid_bootloader: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware flash: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else ifeq ($(strip $(BOOTLOADER)), caterina) - $(call EXEC_AVRDUDE) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_AVRDUDE) else ifeq ($(strip $(BOOTLOADER)), halfkay) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq (dfu,$(findstring dfu,$(BOOTLOADER))) - $(call EXEC_DFU) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU) else ifeq ($(strip $(BOOTLOADER)), USBasp) - $(call EXEC_USBASP) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_USBASP) else ifeq ($(strip $(BOOTLOADER)), bootloadHID) - $(call EXEC_BOOTLOADHID) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_BOOTLOADHID) else ifeq ($(strip $(BOOTLOADER)), qmk-hid) - $(call EXEC_HID_LUFA) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_HID_LUFA) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)" endif diff --git a/platforms/chibios/flash.mk b/platforms/chibios/flash.mk index 46aff6e4b0d..c0b32c2f2b1 100644 --- a/platforms/chibios/flash.mk +++ b/platforms/chibios/flash.mk @@ -73,15 +73,15 @@ teensy: $(BUILD_DIR)/$(TARGET).hex cpfirmware sizeafter flash: $(BUILD_DIR)/$(TARGET).bin cpfirmware sizeafter ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else ifeq ($(strip $(BOOTLOADER)),kiibohd) - $(call EXEC_DFU_UTIL) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL) else ifeq ($(strip $(MCU_FAMILY)),KINETIS) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq ($(strip $(MCU_FAMILY)),MIMXRT1062) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq ($(strip $(MCU_FAMILY)),STM32) - $(call EXEC_DFU_UTIL) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)" endif