varl/qmk_firmware
1
0
Fork 0
forked from forks/qmk_firmware
Code Pull requests Activity

CLI refactoring for common build target APIs (#22221)

Browse source
This commit is contained in:
Nick Brassel 2023-11-15 16:24:54 +11:00 committed by GitHub
parent c4d3521ba6
commit 4938210711
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 296 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

211
lib/python/qmk/build_targets.py Normal file
View file

@ -0,0 +1,211 @@
# Copyright 2023 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
import json
import shutil
from typing import List, Union
from pathlib import Path
from dotty_dict import dotty, Dotty
from milc import cli
from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
from qmk.keyboard import keyboard_folder
from qmk.info import keymap_json
from qmk.cli.generate.compilation_database import write_compilation_database
class BuildTarget:
def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
self._keyboard = keyboard_folder(keyboard)
self._keyboard_safe = self._keyboard.replace('/', '_')
self._keymap = keymap
self._parallel = 1
self._clean = False
self._compiledb = False
self._target = f'{self._keyboard_safe}_{self.keymap}'
self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
self._generated_files_path = self._intermediate_output / 'src'
self._json = json.to_dict() if isinstance(json, Dotty) else json
def __str__(self):
return f'{self.keyboard}:{self.keymap}'
def __repr__(self):
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
if parallel is not None:
self._parallel = parallel
if clean is not None:
self._clean = clean
if compiledb is not None:
self._compiledb = compiledb
@property
def keyboard(self) -> str:
return self._keyboard
@property
def keymap(self) -> str:
return self._keymap
@property
def json(self) -> dict:
if not self._json:
self._load_json()
if not self._json:
return {}
return self._json
@property
def dotty(self) -> Dotty:
return dotty(self.json)
def _common_make_args(self, dry_run: bool = False, build_target: str = None):
compile_args = [
find_make(),
*get_make_parallel_args(self._parallel),
'-r',
'-R',
'-f',
'builddefs/build_keyboard.mk',
]
if not cli.config.general.verbose:
compile_args.append('-s')
verbose = 'true' if cli.config.general.verbose else 'false'
color = 'true' if cli.config.general.color else 'false'
if dry_run:
compile_args.append('-n')
if build_target:
compile_args.append(build_target)
compile_args.extend([
f'KEYBOARD={self.keyboard}',
f'KEYMAP={self.keymap}',
f'KEYBOARD_FILESAFE={self._keyboard_safe}',
f'TARGET={self._target}',
f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
f'VERBOSE={verbose}',
f'COLOR={color}',
'SILENT=false',
'QMK_BIN="qmk"',
])
return compile_args
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
raise NotImplementedError("prepare_build() not implemented in base class")
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
raise NotImplementedError("compile_command() not implemented in base class")
def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
self.prepare_build(build_target=build_target, **env_vars)
command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
if self._clean or self._compiledb:
command = [find_make(), "clean"]
if dry_run:
command.append('-n')
cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
cli.run(command, capture_output=False)
if self._compiledb and not dry_run:
self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
command = self.compile_command(build_target=build_target, **env_vars)
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not dry_run:
cli.echo('\n')
ret = cli.run(command, capture_output=False)
if ret.returncode:
return ret.returncode
class KeyboardKeymapBuildTarget(BuildTarget):
def __init__(self, keyboard: str, keymap: str, json: dict = None):
super().__init__(keyboard=keyboard, keymap=keymap, json=json)
def __repr__(self):
return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
def _load_json(self):
self._json = keymap_json(self.keyboard, self.keymap)
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
pass
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
for key, value in env_vars.items():
compile_args.append(f'{key}={value}')
return compile_args
class JsonKeymapBuildTarget(BuildTarget):
def __init__(self, json_path):
if isinstance(json_path, Path):
self.json_path = json_path
else:
self.json_path = None
json = parse_configurator_json(json_path) # Will load from stdin if provided
# In case the user passes a keymap.json from a keymap directory directly to the CLI.
# e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
json["keymap"] = json.get("keymap", "default_json")
super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
self._keymap_json = self._generated_files_path / 'keymap.json'
def __repr__(self):
return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
def _load_json(self):
pass # Already loaded in constructor
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
if self._clean:
if self._intermediate_output.exists():
shutil.rmtree(self._intermediate_output)
# begin with making the deepest folder in the tree
self._generated_files_path.mkdir(exist_ok=True, parents=True)
# Compare minified to ensure consistent comparison
new_content = json.dumps(self.json, separators=(',', ':'))
if self._keymap_json.exists():
old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
if old_content == new_content:
new_content = None
# Write the keymap.json file if different so timestamps are only updated
# if the content changes -- running `make` won't treat it as modified.
if new_content:
self._keymap_json.write_text(new_content, encoding='utf-8')
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
compile_args.extend([
f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
f'KEYMAP_JSON={self._keymap_json}',
f'KEYMAP_PATH={self._generated_files_path}',
])
for key, value in env_vars.items():
compile_args.append(f'{key}={value}')
return compile_args

4
lib/python/qmk/cli/clean.py
View file

@ -2,7 +2,7 @@
"""
from subprocess import DEVNULL
from qmk.commands import create_make_target
from qmk.commands import find_make
from milc import cli
@ -11,4 +11,4 @@ from milc import cli
def clean(cli):
"""Runs `make clean` (or `make distclean` if --all is passed)
"""
cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)

58
lib/python/qmk/cli/compile.py
View file

@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter
from milc import cli
import qmk.path
from qmk.constants import QMK_FIRMWARE
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
from qmk.commands import build_environment
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
from qmk.keymap import keymap_completer, locate_keymap
from qmk.cli.generate.compilation_database import write_compilation_database
def _is_keymap_target(keyboard, keymap):
if keymap == 'all':
return True
if locate_keymap(keyboard, keymap):
return True
return False
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
@cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
@cli.subcommand('Compile a QMK Firmware.')
@automagic_keyboard
@ -53,47 +43,27 @@ def compile(cli):
# Build the environment vars
envs = build_environment(cli.args.env)
# Determine the compile command
commands = []
current_keyboard = None
current_keymap = None
# Handler for the build target
target = None
if cli.args.filename:
# If a configurator JSON was provided generate a keymap and compile it
user_keymap = parse_configurator_json(cli.args.filename)
commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
# if we were given a filename, assume we have a json build target
target = JsonKeymapBuildTarget(cli.args.filename)
elif cli.config.compile.keyboard and cli.config.compile.keymap:
# Generate the make command for a specific keyboard/keymap.
if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
# if we got a keyboard and keymap, attempt to find it
if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
cli.log.error('Invalid keymap argument.')
cli.print_help()
return False
if cli.args.clean:
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
# If we got here, then we have a valid keyboard and keymap for a build target
target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
current_keyboard = cli.config.compile.keyboard
current_keymap = cli.config.compile.keymap
if not commands:
if not target:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.print_help()
return False
if cli.args.compiledb:
if current_keyboard is None or current_keymap is None:
cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
cli.print_help()
return False
write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
if not cli.args.dry_run:
cli.echo('\n')
for command in commands:
ret = cli.run(command, capture_output=False)
if ret.returncode:
return ret.returncode
target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)

14
lib/python/qmk/cli/find.py
View file

@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets
def find(cli):
"""Search through all keyboards and keymaps for a given search criteria.
"""
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
print(f'{target}')
if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
for keyboard, keymap, print_vals in targets:
print(f'{keyboard}:{keymap}')
for key, val in print_vals:
print(f' {key}={val}')
for key in cli.args.print:
print(f' {key}={target.dotty.get(key, None)}')

45
lib/python/qmk/cli/flash.py
View file

@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
A bootloader must be specified.
"""
from argcomplete.completers import FilesCompleter
from pathlib import Path
from milc import cli
import qmk.path
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
from qmk.commands import build_environment
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.keymap import keymap_completer, locate_keymap
from qmk.flashers import flasher
def _is_keymap_target(keyboard, keymap):
if keymap == 'all':
return True
if locate_keymap(keyboard, keymap):
return True
return False
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
def _list_bootloaders():
@ -89,7 +81,7 @@ def flash(cli):
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
"""
if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
return _flash_binary(cli.args.filename, cli.args.mcu)
if cli.args.bootloaders:
@ -98,34 +90,27 @@ def flash(cli):
# Build the environment vars
envs = build_environment(cli.args.env)
# Determine the compile command
commands = []
# Handler for the build target
target = None
if cli.args.filename:
# If a configurator JSON was provided generate a keymap and compile it
user_keymap = parse_configurator_json(cli.args.filename)
commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
# if we were given a filename, assume we have a json build target
target = JsonKeymapBuildTarget(cli.args.filename)
elif cli.config.flash.keyboard and cli.config.flash.keymap:
# Generate the make command for a specific keyboard/keymap.
if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
# if we got a keyboard and keymap, attempt to find it
if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
cli.log.error('Invalid keymap argument.')
cli.print_help()
return False
if cli.args.clean:
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
# If we got here, then we have a valid keyboard and keymap for a build target
target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
if not commands:
if not target:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.print_help()
return False
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
if not cli.args.dry_run:
cli.echo('\n')
for command in commands:
ret = cli.run(command, capture_output=False)
if ret.returncode:
return ret.returncode
target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)

16
lib/python/qmk/cli/generate/compilation_database.py
View file

@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union
from milc import cli, MILC
from qmk.commands import create_make_command
from qmk.commands import find_make
from qmk.constants import QMK_FIRMWARE
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.keyboard import keyboard_completer, keyboard_folder
@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
return records
def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
# Generate the make command for a specific keyboard/keymap.
command = create_make_command(keyboard, keymap, dry_run=True)
if not command:
from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references
target = KeyboardKeymapBuildTarget(keyboard, keymap)
command = target.compile_command(dry_run=True, **env_vars)
if not command:
cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
env.pop("MAKEFLAGS", None)
# re-use same executable as the main make invocation (might be gmake)
clean_command = [command[0], 'clean']
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
cli.run(clean_command, capture_output=False, check=True, env=env)
if not skip_clean:
clean_command = [find_make(), "clean"]
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
cli.run(clean_command, capture_output=False, check=True, env=env)
cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))

29
lib/python/qmk/cli/mass_compile.py
View file

@ -3,26 +3,28 @@
This will compile everything in parallel, for testing purposes.
"""
import os
from typing import List
from pathlib import Path
from subprocess import DEVNULL
from milc import cli
from qmk.constants import QMK_FIRMWARE
from qmk.commands import _find_make, get_make_parallel_args
from qmk.commands import find_make, get_make_parallel_args, build_environment
from qmk.search import search_keymap_targets, search_make_targets
from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
if len(targets) == 0:
return
make_cmd = _find_make()
make_cmd = find_make()
builddir = Path(QMK_FIRMWARE) / '.build'
makefile = builddir / 'parallel_kb_builds.mk'
if dry_run:
cli.log.info('Compilation targets:')
for target in sorted(targets):
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
else:
if clean:
@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
builddir.mkdir(parents=True, exist_ok=True)
with open(makefile, "w") as f:
for target in sorted(targets):
keyboard_name = target[0]
keymap_name = target[1]
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
keyboard_name = target.keyboard
keymap_name = target.keymap
target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
command = target.compile_command(**env)
command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
keyboard_safe = keyboard_name.replace('/', '_')
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary
{keyboard_safe}_{keymap_name}_binary:
@rm -f "{build_log}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
{' '.join(command)} \\
>>"{build_log}" 2>&1 \\
|| cp "{build_log}" "{failed_log}"
@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
@ -95,8 +101,11 @@ def mass_compile(cli):
"""Compile QMK Firmware against all keyboards.
"""
if len(cli.args.builds) > 0:
targets = search_make_targets(cli.args.builds, cli.args.filter)
json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
targets = search_make_targets(make_like_targets)
targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
else:
targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))

164
lib/python/qmk/commands.py
View file

@ -2,18 +2,16 @@
"""
import os
import sys
import json
import shutil
from pathlib import Path
from milc import cli
import jsonschema
from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
from qmk.json_schema import json_load, validate
def _find_make():
def find_make():
"""Returns the correct make command for this environment.
"""
make_cmd = os.environ.get('MAKE')
@ -24,74 +22,6 @@ def _find_make():
return make_cmd
def create_make_target(target, dry_run=False, parallel=1, **env_vars):
"""Create a make command
Args:
target
Usually a make rule, such as 'clean' or 'all'.
dry_run
make -n -- don't actually build
parallel
The number of make jobs to run in parallel
**env_vars
Environment variables to be passed to make.
Returns:
A command that can be run to make the specified keyboard and keymap
"""
env = []
make_cmd = _find_make()
for key, value in env_vars.items():
env.append(f'{key}={value}')
if cli.config.general.verbose:
env.append('VERBOSE=true')
return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
"""Create a make compile command
Args:
keyboard
The path of the keyboard, for example 'plank'
keymap
The name of the keymap, for example 'algernon'
target
Usually a bootloader.
dry_run
make -n -- don't actually build
parallel
The number of make jobs to run in parallel
**env_vars
Environment variables to be passed to make.
Returns:
A command that can be run to make the specified keyboard and keymap
"""
make_args = [keyboard, keymap]
if target:
make_args.append(target)
return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
def get_make_parallel_args(parallel=1):
"""Returns the arguments for running the specified number of parallel jobs.
"""
@ -100,7 +30,7 @@ def get_make_parallel_args(parallel=1):
if int(parallel) <= 0:
# 0 or -1 means -j without argument (unlimited jobs)
parallel_args.append('--jobs')
else:
elif int(parallel) > 1:
parallel_args.append('--jobs=' + str(parallel))
if int(parallel) != 1:
@ -110,96 +40,6 @@ def get_make_parallel_args(parallel=1):
return parallel_args
def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
"""Convert a configurator export JSON file into a C file and then compile it.
Args:
user_keymap
A deserialized keymap export
bootloader
A bootloader to flash
parallel
The number of make jobs to run in parallel
Returns:
A command to run to compile and flash the C file.
"""
# In case the user passes a keymap.json from a keymap directory directly to the CLI.
# e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
keymap_dir = intermediate_output / 'src'
keymap_json = keymap_dir / 'keymap.json'
if clean:
if intermediate_output.exists():
shutil.rmtree(intermediate_output)
# begin with making the deepest folder in the tree
keymap_dir.mkdir(exist_ok=True, parents=True)
# Compare minified to ensure consistent comparison
new_content = json.dumps(user_keymap, separators=(',', ':'))
if keymap_json.exists():
old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
if old_content == new_content:
new_content = None
# Write the keymap.json file if different
if new_content:
keymap_json.write_text(new_content, encoding='utf-8')
# Return a command that can be run to make the keymap and flash if given
verbose = 'true' if cli.config.general.verbose else 'false'
color = 'true' if cli.config.general.color else 'false'
make_command = [_find_make()]
if not cli.config.general.verbose:
make_command.append('-s')
make_command.extend([
*get_make_parallel_args(parallel),
'-r',
'-R',
'-f',
'builddefs/build_keyboard.mk',
])
if bootloader:
make_command.append(bootloader)
make_command.extend([
f'KEYBOARD={user_keymap["keyboard"]}',
f'KEYMAP={user_keymap["keymap"]}',
f'KEYBOARD_FILESAFE={keyboard_filesafe}',
f'TARGET={target}',
f'INTERMEDIATE_OUTPUT={intermediate_output}',
f'MAIN_KEYMAP_PATH_1={intermediate_output}',
f'MAIN_KEYMAP_PATH_2={intermediate_output}',
f'MAIN_KEYMAP_PATH_3={intermediate_output}',
f'MAIN_KEYMAP_PATH_4={intermediate_output}',
f'MAIN_KEYMAP_PATH_5={intermediate_output}',
f'KEYMAP_JSON={keymap_json}',
f'KEYMAP_PATH={keymap_dir}',
f'VERBOSE={verbose}',
f'COLOR={color}',
'SILENT=false',
'QMK_BIN="qmk"',
])
for key, value in env_vars.items():
make_command.append(f'{key}={value}')
return make_command
def parse_configurator_json(configurator_file):
"""Open and parse a configurator json export
"""

6
lib/python/qmk/keyboard.py
View file

@ -98,11 +98,7 @@ def keyboard_folder(keyboard):
if keyboard == last_keyboard:
break
rules_mk_file = Path(base_path, keyboard, 'rules.mk')
if rules_mk_file.exists():
rules_mk = parse_rules_mk_file(rules_mk_file)
keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard)
keyboard = resolve_keyboard(keyboard)
if not qmk.path.is_keyboard(keyboard):
raise ValueError(f'Invalid keyboard: {keyboard}')

34
lib/python/qmk/search.py
View file

@ -11,8 +11,9 @@ from milc import cli
from qmk.util import parallel_map
from qmk.info import keymap_json
import qmk.keyboard
import qmk.keymap
from qmk.keyboard import list_keyboards, keyboard_folder
from qmk.keymap import list_keymaps, locate_keymap
from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
def _set_log_level(level):
@ -36,15 +37,15 @@ def _all_keymaps(keyboard):
"""Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
"""
with ignore_logging():
keyboard = qmk.keyboard.resolve_keyboard(keyboard)
return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)]
keyboard = keyboard_folder(keyboard)
return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
def _keymap_exists(keyboard, keymap):
"""Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.
"""
with ignore_logging():
return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
return keyboard if locate_keymap(keyboard, keymap) is not None else None
def _load_keymap_info(kb_km):
@ -75,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
Caters for 'all' in either keyboard or keymap, or both.
"""
if all_keyboards is None:
all_keyboards = qmk.keyboard.list_keyboards()
all_keyboards = list_keyboards()
if keyboard == 'all':
if keymap == 'all':
@ -90,30 +91,29 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
else:
if keymap == 'all':
keyboard = qmk.keyboard.resolve_keyboard(keyboard)
cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
return _all_keymaps(keyboard)
else:
return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
return [(keyboard, keymap)]
def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
"""Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
"""
overall_targets = []
all_keyboards = qmk.keyboard.list_keyboards()
all_keyboards = list_keyboards()
for target in targets:
overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
return list(sorted(set(overall_targets)))
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
Optionally includes the values of the queried info.json keys.
"""
if len(filters) == 0 and len(print_vals) == 0:
targets = [(kb, km, {}) for kb, km in target_list]
if len(filters) == 0:
targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@ -172,18 +172,18 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
continue
targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
return targets
def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
return _filter_keymap_targets(expand_keymap_targets(targets), filters)
def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
return _filter_keymap_targets(expand_make_targets(targets), filters)