cmake: improved board handling for revisions

This commit improves board handling for boards in HWMv2.
On a CMake rerun, then BOARD_DIR is passed to `list_boards.py` which
is extended to take such parameter.

This allows to run `list_boards.py` whenever CMake reruns without the
penalty of searching for all board.yml files, as only the board.yml of
the current BOARD_DIR is processed.

This allows `list_boards.py` to be invoked and from there obtain list
of valid revisions and board identifiers for further board validation.

This removes the need for caching additional CMake variables related to
the board identifier and revision and thereby remove the risk of
settings becoming out of sync as only the board provided by user is
needed.

This work further ensure that use-cases described in #50536 is still
supported.

Signed-off-by: Torsten Rasmussen <Torsten.Rasmussen@nordicsemi.no>

cmake/modules/boards.cmake

@@ -115,7 +115,7 @@ Hints:
 endforeach()
 
 if((HWMv1 AND NOT EXISTS ${BOARD_DIR}/${BOARD}_defconfig)
-   OR (HWMv2 AND NOT EXISTS ${BOARD_DIR}))
+   OR (HWMv2 AND NOT EXISTS ${BOARD_DIR}/board.yml))
   message(WARNING "BOARD_DIR: ${BOARD_DIR} has been moved or deleted. "
                   "Trying to find new location."
   )
@@ -152,20 +152,26 @@ if(NOT BOARD_DIR)
       message("Board alias ${BOARD_ALIAS} is hiding the real board of same name")
     endif()
   endif()
+endif()
 
-  set(format_str "{NAME}\;{DIR}\;{HWM}\;")
-  set(format_str "${format_str}{REVISION_FORMAT}\;{REVISION_DEFAULT}\;{REVISION_EXACT}\;")
-  set(format_str "${format_str}{REVISIONS}\;{SOCS}\;{IDENTIFIERS}")
+set(format_str "{NAME}\;{DIR}\;{HWM}\;")
+set(format_str "${format_str}{REVISION_FORMAT}\;{REVISION_DEFAULT}\;{REVISION_EXACT}\;")
+set(format_str "${format_str}{REVISIONS}\;{SOCS}\;{IDENTIFIERS}")
 
-  execute_process(${list_boards_commands} --board=${BOARD}
-    --cmakeformat=${format_str}
-                  OUTPUT_VARIABLE ret_board
-                  ERROR_VARIABLE err_board
-                  RESULT_VARIABLE ret_val
-  )
-  if(ret_val)
-    message(FATAL_ERROR "Error finding board: ${BOARD}\nError message: ${err_board}")
-  endif()
+if(BOARD_DIR)
+  set(board_dir_arg "--board-dir=${BOARD_DIR}")
+endif()
+execute_process(${list_boards_commands} --board=${BOARD} ${board_dir_arg}
+  --cmakeformat=${format_str}
+                OUTPUT_VARIABLE ret_board
+                ERROR_VARIABLE err_board
+                RESULT_VARIABLE ret_val
+)
+if(ret_val)
+  message(FATAL_ERROR "Error finding board: ${BOARD}\nError message: ${err_board}")
+endif()
+
+if(NOT "${ret_board}" STREQUAL "")
   string(STRIP "${ret_board}" ret_board)
   set(single_val "NAME;DIR;HWM;REVISION_FORMAT;REVISION_DEFAULT;REVISION_EXACT")
   set(multi_val  "REVISIONS;SOCS;IDENTIFIERS")
@@ -178,9 +184,12 @@ if(NOT BOARD_DIR)
   # CMake variable: HWMv2=True, when HWMv2 is in use.
   set(HWM       ${BOARD_HWM} CACHE INTERNAL "Zephyr hardware model version")
   set(HWM${HWM} True   CACHE INTERNAL "Zephyr hardware model")
-endif()
-
-if(NOT BOARD_DIR)
+elseif(BOARD_DIR)
+  message(FATAL_ERROR "Error finding board: ${BOARD} in ${BOARD_DIR}.\n"
+          "This indicates the board has been removed, renamed, or placed at a new location.\n"
+	  "Please run a pristine build."
+  )
+else()
   message("No board named '${BOARD}' found.\n\n"
           "Please choose one of the following boards:\n"
   )
@@ -260,7 +269,6 @@ elseif(HWMv2)
             `${BOARD}` not found. Please specify a valid board.\n"
             "Valid board identifiers for ${BOARD_NAME} are:\n${BOARD_IDENTIFIERS}\n")
     endif()
-    set(BOARD_IDENTIFIER ${BOARD_IDENTIFIER} CACHE INTERNAL "Board identifier")
   endif()
 else()
   message(FATAL_ERROR "Unknown hw model (${HWM}) for board: ${BOARD}.")

scripts/ci/test_plan.py

@@ -226,7 +226,8 @@ class Filters:
             roots.append(repository_path)
 
         # Look for boards in monitored repositories
-        lb_args = argparse.Namespace(**{ 'arch_roots': roots, 'board_roots': roots, 'board': None})
+        lb_args = argparse.Namespace(**{'arch_roots': roots, 'board_roots': roots, 'board': None,
+                                        'board_dir': None})
         known_boards = list_boards.find_boards(lb_args)
         for b in boards:
             name_re = re.compile(b)

scripts/list_boards.py

@@ -114,7 +114,7 @@ def find_arch2board_set(args):
     ret = defaultdict(set)
 
     for root in args.board_roots:
-        for arch, boards in find_arch2board_set_in(root, arches).items():
+        for arch, boards in find_arch2board_set_in(root, arches, args.board_dir).items():
             if args.board is not None:
                 ret[arch] |= {b for b in boards if b.name == args.board}
             else:
@@ -145,7 +145,7 @@ def find_arches_in(root):
 
     return ret
 
-def find_arch2board_set_in(root, arches):
+def find_arch2board_set_in(root, arches, board_dir):
     ret = defaultdict(set)
     boards = root / 'boards'
 
@@ -156,6 +156,8 @@ def find_arch2board_set_in(root, arches):
         for maybe_board in (boards / "boards_legacy" / arch).iterdir():
             if not maybe_board.is_dir():
                 continue
+            if board_dir is not None and board_dir != maybe_board:
+                continue
             for maybe_defconfig in maybe_board.iterdir():
                 file_name = maybe_defconfig.name
                 if file_name.endswith('_defconfig'):
@@ -165,6 +167,63 @@ def find_arch2board_set_in(root, arches):
     return ret
 
 
+def load_v2_boards(board_name, board_yml, systems):
+    boards = []
+    if board_yml.is_file():
+        with board_yml.open('r') as f:
+            b = yaml.safe_load(f.read())
+
+        try:
+            pykwalify.core.Core(source_data=b, schema_data=board_schema).validate()
+        except pykwalify.errors.SchemaError as e:
+            sys.exit('ERROR: Malformed "build" section in file: {}\n{}'
+                     .format(board_yml.as_posix(), e))
+
+        mutual_exclusive = {'board', 'boards'}
+        if len(mutual_exclusive - b.keys()) < 1:
+            sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n'
+                     f'{mutual_exclusive} are mutual exclusive at this level.')
+
+        board_array = b.get('boards', [ b.get('board', None) ])
+        for board in board_array:
+            if board_name is not None:
+                if board['name'] != board_name:
+                    # Not the board we're looking for, ignore.
+                    continue
+
+            board_revision = board.get('revision')
+            if board_revision is not None and board_revision.get('format') != 'custom':
+                if board_revision.get('default') is None:
+                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
+                             "Cannot find required key 'default'. Path: '/board/revision.'")
+                if board_revision.get('revisions') is None:
+                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
+                             "Cannot find required key 'revisions'. Path: '/board/revision.'")
+
+            mutual_exclusive = {'socs', 'variants'}
+            if len(mutual_exclusive - board.keys()) < 1:
+                sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
+                         f'{mutual_exclusive} are mutual exclusive at this level.')
+            socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', []))
+                    for s in board.get('socs', {})]
+
+            board = Board(
+                name=board['name'],
+                dir=board_yml.parent,
+                vendor=board.get('vendor'),
+                revision_format=board.get('revision', {}).get('format'),
+                revision_default=board.get('revision', {}).get('default'),
+                revision_exact=board.get('revision', {}).get('exact', False),
+                revisions=[Revision.from_dict(v) for v in
+                           board.get('revision', {}).get('revisions', [])],
+                socs=socs,
+                variants=[Variant.from_dict(v) for v in board.get('variants', [])],
+                hwm='v2',
+            )
+            boards.append(board)
+    return boards
+
+
 def find_v2_boards(args):
     root_args = argparse.Namespace(**{'soc_roots': args.soc_roots})
     systems = list_hardware.find_v2_systems(root_args)
@@ -175,58 +234,8 @@ def find_v2_boards(args):
         board_files.extend((root / 'boards').rglob(BOARD_YML))
 
     for board_yml in board_files:
-        if board_yml.is_file():
-            with board_yml.open('r') as f:
-                b = yaml.safe_load(f.read())
-
-            try:
-                pykwalify.core.Core(source_data=b, schema_data=board_schema).validate()
-            except pykwalify.errors.SchemaError as e:
-                sys.exit('ERROR: Malformed "build" section in file: {}\n{}'
-                         .format(board_yml.as_posix(), e))
-
-            mutual_exclusive = {'board', 'boards'}
-            if len(mutual_exclusive - b.keys()) < 1:
-                sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
-
-            board_array = b.get('boards', [ b.get('board', None) ])
-            for board in board_array:
-                if args.board is not None:
-                    if board['name'] != args.board:
-                        # Not the board we're looking for, ignore.
-                        continue
-
-                board_revision = board.get('revision')
-                if board_revision is not None and board_revision.get('format') != 'custom':
-                    if board_revision.get('default') is None:
-                        sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                                 "Cannot find required key 'default'. Path: '/board/revision.'")
-                    if board_revision.get('revisions') is None:
-                        sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                                 "Cannot find required key 'revisions'. Path: '/board/revision.'")
-
-                mutual_exclusive = {'socs', 'variants'}
-                if len(mutual_exclusive - board.keys()) < 1:
-                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                             f'{mutual_exclusive} are mutual exclusive at this level.')
-                socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', []))
-                        for s in board.get('socs', {})]
-
-                board = Board(
-                    name=board['name'],
-                    dir=board_yml.parent,
-                    vendor=board.get('vendor'),
-                    revision_format=board.get('revision', {}).get('format'),
-                    revision_default=board.get('revision', {}).get('default'),
-                    revision_exact=board.get('revision', {}).get('exact', False),
-                    revisions=[Revision.from_dict(v) for v in
-                               board.get('revision', {}).get('revisions', [])],
-                    socs=socs,
-                    variants=[Variant.from_dict(v) for v in board.get('variants', [])],
-                    hwm='v2',
-                )
-                boards.append(board)
+        b = load_v2_boards(args.board, board_yml, systems)
+        boards.extend(b)
     return boards
 
 
@@ -251,6 +260,8 @@ def add_args(parser):
                         help='add a soc root, may be given more than once')
     parser.add_argument("--board", dest='board', default=None,
                         help='lookup the specific board, fail if not found')
+    parser.add_argument("--board-dir", default=None, type=Path,
+                        help='Only look for boards at the specific location')
 
 
 def add_args_formatting(parser):
@@ -290,7 +301,12 @@ def board_v2_identifiers(board):
 
 
 def dump_v2_boards(args):
-    boards = find_v2_boards(args)
+    if args.board_dir:
+        root_args = argparse.Namespace(**{'soc_roots': args.soc_roots})
+        systems = list_hardware.find_v2_systems(root_args)
+        boards = load_v2_boards(args.board, args.board_dir / BOARD_YML, systems)
+    else:
+        boards = find_v2_boards(args)
 
     for b in boards:
         identifiers = board_v2_identifiers(b)

scripts/pylib/twister/twisterlib/testplan.py

@@ -405,7 +405,7 @@ class TestPlan:
         # but in Zephyr build system, the board root is without the `boards` in folder path.
         board_roots = [Path(os.path.dirname(root)) for root in self.env.board_roots]
         lb_args = Namespace(arch_roots=[Path(ZEPHYR_BASE)], soc_roots=[Path(ZEPHYR_BASE)],
-                            board_roots=board_roots, board=None)
+                            board_roots=board_roots, board=None, board_dir=None)
         v1_boards = list_boards.find_boards(lb_args)
         v2_boards = list_boards.find_v2_boards(lb_args)
         for b in v1_boards: