From 22dc2b6391c319f934160aa858a94d584f8561c7 Mon Sep 17 00:00:00 2001 From: Torsten Rasmussen Date: Wed, 7 Feb 2024 16:04:50 +0100 Subject: [PATCH] cmake: improved board handling for revisions This commit improves board handling for boards in HWMv2. On a CMake rerun, then BOARD_DIR is passed to `list_boards.py` which is extended to take such parameter. This allows to run `list_boards.py` whenever CMake reruns without the penalty of searching for all board.yml files, as only the board.yml of the current BOARD_DIR is processed. This allows `list_boards.py` to be invoked and from there obtain list of valid revisions and board identifiers for further board validation. This removes the need for caching additional CMake variables related to the board identifier and revision and thereby remove the risk of settings becoming out of sync as only the board provided by user is needed. This work further ensure that use-cases described in #50536 is still supported. Signed-off-by: Torsten Rasmussen --- cmake/modules/boards.cmake | 42 ++++--- scripts/ci/test_plan.py | 3 +- scripts/list_boards.py | 126 +++++++++++-------- scripts/pylib/twister/twisterlib/testplan.py | 2 +- 4 files changed, 99 insertions(+), 74 deletions(-) diff --git a/cmake/modules/boards.cmake b/cmake/modules/boards.cmake index 974108c31e2..3a66fd709cd 100644 --- a/cmake/modules/boards.cmake +++ b/cmake/modules/boards.cmake @@ -115,7 +115,7 @@ Hints: endforeach() if((HWMv1 AND NOT EXISTS ${BOARD_DIR}/${BOARD}_defconfig) - OR (HWMv2 AND NOT EXISTS ${BOARD_DIR})) + OR (HWMv2 AND NOT EXISTS ${BOARD_DIR}/board.yml)) message(WARNING "BOARD_DIR: ${BOARD_DIR} has been moved or deleted. " "Trying to find new location." ) @@ -152,20 +152,26 @@ if(NOT BOARD_DIR) message("Board alias ${BOARD_ALIAS} is hiding the real board of same name") endif() endif() +endif() - set(format_str "{NAME}\;{DIR}\;{HWM}\;") - set(format_str "${format_str}{REVISION_FORMAT}\;{REVISION_DEFAULT}\;{REVISION_EXACT}\;") - set(format_str "${format_str}{REVISIONS}\;{SOCS}\;{IDENTIFIERS}") +set(format_str "{NAME}\;{DIR}\;{HWM}\;") +set(format_str "${format_str}{REVISION_FORMAT}\;{REVISION_DEFAULT}\;{REVISION_EXACT}\;") +set(format_str "${format_str}{REVISIONS}\;{SOCS}\;{IDENTIFIERS}") - execute_process(${list_boards_commands} --board=${BOARD} - --cmakeformat=${format_str} - OUTPUT_VARIABLE ret_board - ERROR_VARIABLE err_board - RESULT_VARIABLE ret_val - ) - if(ret_val) - message(FATAL_ERROR "Error finding board: ${BOARD}\nError message: ${err_board}") - endif() +if(BOARD_DIR) + set(board_dir_arg "--board-dir=${BOARD_DIR}") +endif() +execute_process(${list_boards_commands} --board=${BOARD} ${board_dir_arg} + --cmakeformat=${format_str} + OUTPUT_VARIABLE ret_board + ERROR_VARIABLE err_board + RESULT_VARIABLE ret_val +) +if(ret_val) + message(FATAL_ERROR "Error finding board: ${BOARD}\nError message: ${err_board}") +endif() + +if(NOT "${ret_board}" STREQUAL "") string(STRIP "${ret_board}" ret_board) set(single_val "NAME;DIR;HWM;REVISION_FORMAT;REVISION_DEFAULT;REVISION_EXACT") set(multi_val "REVISIONS;SOCS;IDENTIFIERS") @@ -178,9 +184,12 @@ if(NOT BOARD_DIR) # CMake variable: HWMv2=True, when HWMv2 is in use. set(HWM ${BOARD_HWM} CACHE INTERNAL "Zephyr hardware model version") set(HWM${HWM} True CACHE INTERNAL "Zephyr hardware model") -endif() - -if(NOT BOARD_DIR) +elseif(BOARD_DIR) + message(FATAL_ERROR "Error finding board: ${BOARD} in ${BOARD_DIR}.\n" + "This indicates the board has been removed, renamed, or placed at a new location.\n" + "Please run a pristine build." + ) +else() message("No board named '${BOARD}' found.\n\n" "Please choose one of the following boards:\n" ) @@ -260,7 +269,6 @@ elseif(HWMv2) `${BOARD}` not found. Please specify a valid board.\n" "Valid board identifiers for ${BOARD_NAME} are:\n${BOARD_IDENTIFIERS}\n") endif() - set(BOARD_IDENTIFIER ${BOARD_IDENTIFIER} CACHE INTERNAL "Board identifier") endif() else() message(FATAL_ERROR "Unknown hw model (${HWM}) for board: ${BOARD}.") diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index ee8028d5cc3..26787496a7c 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -226,7 +226,8 @@ class Filters: roots.append(repository_path) # Look for boards in monitored repositories - lb_args = argparse.Namespace(**{ 'arch_roots': roots, 'board_roots': roots, 'board': None}) + lb_args = argparse.Namespace(**{'arch_roots': roots, 'board_roots': roots, 'board': None, + 'board_dir': None}) known_boards = list_boards.find_boards(lb_args) for b in boards: name_re = re.compile(b) diff --git a/scripts/list_boards.py b/scripts/list_boards.py index 79aefa8bcae..cfe6f0c7393 100755 --- a/scripts/list_boards.py +++ b/scripts/list_boards.py @@ -114,7 +114,7 @@ def find_arch2board_set(args): ret = defaultdict(set) for root in args.board_roots: - for arch, boards in find_arch2board_set_in(root, arches).items(): + for arch, boards in find_arch2board_set_in(root, arches, args.board_dir).items(): if args.board is not None: ret[arch] |= {b for b in boards if b.name == args.board} else: @@ -145,7 +145,7 @@ def find_arches_in(root): return ret -def find_arch2board_set_in(root, arches): +def find_arch2board_set_in(root, arches, board_dir): ret = defaultdict(set) boards = root / 'boards' @@ -156,6 +156,8 @@ def find_arch2board_set_in(root, arches): for maybe_board in (boards / "boards_legacy" / arch).iterdir(): if not maybe_board.is_dir(): continue + if board_dir is not None and board_dir != maybe_board: + continue for maybe_defconfig in maybe_board.iterdir(): file_name = maybe_defconfig.name if file_name.endswith('_defconfig'): @@ -165,6 +167,63 @@ def find_arch2board_set_in(root, arches): return ret +def load_v2_boards(board_name, board_yml, systems): + boards = [] + if board_yml.is_file(): + with board_yml.open('r') as f: + b = yaml.safe_load(f.read()) + + try: + pykwalify.core.Core(source_data=b, schema_data=board_schema).validate() + except pykwalify.errors.SchemaError as e: + sys.exit('ERROR: Malformed "build" section in file: {}\n{}' + .format(board_yml.as_posix(), e)) + + mutual_exclusive = {'board', 'boards'} + if len(mutual_exclusive - b.keys()) < 1: + sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n' + f'{mutual_exclusive} are mutual exclusive at this level.') + + board_array = b.get('boards', [ b.get('board', None) ]) + for board in board_array: + if board_name is not None: + if board['name'] != board_name: + # Not the board we're looking for, ignore. + continue + + board_revision = board.get('revision') + if board_revision is not None and board_revision.get('format') != 'custom': + if board_revision.get('default') is None: + sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' + "Cannot find required key 'default'. Path: '/board/revision.'") + if board_revision.get('revisions') is None: + sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' + "Cannot find required key 'revisions'. Path: '/board/revision.'") + + mutual_exclusive = {'socs', 'variants'} + if len(mutual_exclusive - board.keys()) < 1: + sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' + f'{mutual_exclusive} are mutual exclusive at this level.') + socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', [])) + for s in board.get('socs', {})] + + board = Board( + name=board['name'], + dir=board_yml.parent, + vendor=board.get('vendor'), + revision_format=board.get('revision', {}).get('format'), + revision_default=board.get('revision', {}).get('default'), + revision_exact=board.get('revision', {}).get('exact', False), + revisions=[Revision.from_dict(v) for v in + board.get('revision', {}).get('revisions', [])], + socs=socs, + variants=[Variant.from_dict(v) for v in board.get('variants', [])], + hwm='v2', + ) + boards.append(board) + return boards + + def find_v2_boards(args): root_args = argparse.Namespace(**{'soc_roots': args.soc_roots}) systems = list_hardware.find_v2_systems(root_args) @@ -175,58 +234,8 @@ def find_v2_boards(args): board_files.extend((root / 'boards').rglob(BOARD_YML)) for board_yml in board_files: - if board_yml.is_file(): - with board_yml.open('r') as f: - b = yaml.safe_load(f.read()) - - try: - pykwalify.core.Core(source_data=b, schema_data=board_schema).validate() - except pykwalify.errors.SchemaError as e: - sys.exit('ERROR: Malformed "build" section in file: {}\n{}' - .format(board_yml.as_posix(), e)) - - mutual_exclusive = {'board', 'boards'} - if len(mutual_exclusive - b.keys()) < 1: - sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n' - f'{mutual_exclusive} are mutual exclusive at this level.') - - board_array = b.get('boards', [ b.get('board', None) ]) - for board in board_array: - if args.board is not None: - if board['name'] != args.board: - # Not the board we're looking for, ignore. - continue - - board_revision = board.get('revision') - if board_revision is not None and board_revision.get('format') != 'custom': - if board_revision.get('default') is None: - sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' - "Cannot find required key 'default'. Path: '/board/revision.'") - if board_revision.get('revisions') is None: - sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' - "Cannot find required key 'revisions'. Path: '/board/revision.'") - - mutual_exclusive = {'socs', 'variants'} - if len(mutual_exclusive - board.keys()) < 1: - sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n' - f'{mutual_exclusive} are mutual exclusive at this level.') - socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', [])) - for s in board.get('socs', {})] - - board = Board( - name=board['name'], - dir=board_yml.parent, - vendor=board.get('vendor'), - revision_format=board.get('revision', {}).get('format'), - revision_default=board.get('revision', {}).get('default'), - revision_exact=board.get('revision', {}).get('exact', False), - revisions=[Revision.from_dict(v) for v in - board.get('revision', {}).get('revisions', [])], - socs=socs, - variants=[Variant.from_dict(v) for v in board.get('variants', [])], - hwm='v2', - ) - boards.append(board) + b = load_v2_boards(args.board, board_yml, systems) + boards.extend(b) return boards @@ -251,6 +260,8 @@ def add_args(parser): help='add a soc root, may be given more than once') parser.add_argument("--board", dest='board', default=None, help='lookup the specific board, fail if not found') + parser.add_argument("--board-dir", default=None, type=Path, + help='Only look for boards at the specific location') def add_args_formatting(parser): @@ -290,7 +301,12 @@ def board_v2_identifiers(board): def dump_v2_boards(args): - boards = find_v2_boards(args) + if args.board_dir: + root_args = argparse.Namespace(**{'soc_roots': args.soc_roots}) + systems = list_hardware.find_v2_systems(root_args) + boards = load_v2_boards(args.board, args.board_dir / BOARD_YML, systems) + else: + boards = find_v2_boards(args) for b in boards: identifiers = board_v2_identifiers(b) diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index c5864b9dc26..f14180dd09b 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -405,7 +405,7 @@ class TestPlan: # but in Zephyr build system, the board root is without the `boards` in folder path. board_roots = [Path(os.path.dirname(root)) for root in self.env.board_roots] lb_args = Namespace(arch_roots=[Path(ZEPHYR_BASE)], soc_roots=[Path(ZEPHYR_BASE)], - board_roots=board_roots, board=None) + board_roots=board_roots, board=None, board_dir=None) v1_boards = list_boards.find_boards(lb_args) v2_boards = list_boards.find_v2_boards(lb_args) for b in v1_boards: