From fb055ea77e007cd044b1cf06b80d08ed804e1bf4 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 12 Nov 2025 11:34:06 -0800 Subject: [PATCH] logging: dictionary: format scripts with ruff Simple formatting via ruff on dictionary logging scripts. No manual editing was done on the scripts. Signed-off-by: Daniel Leung --- .ruff-excludes.toml | 10 -- scripts/logging/dictionary/database_gen.py | 116 +++++++++--------- .../dictionary_parser/data_types.py | 19 +-- .../dictionary_parser/log_database.py | 55 +++------ .../dictionary_parser/log_parser.py | 5 +- .../dictionary_parser/log_parser_v1.py | 53 ++++---- .../dictionary_parser/log_parser_v3.py | 58 ++++----- .../dictionary/dictionary_parser/utils.py | 2 +- scripts/logging/dictionary/log_parser.py | 14 ++- scripts/logging/dictionary/log_parser_uart.py | 12 +- .../pytest/test_logging_dictionary.py | 93 +++++++------- 11 files changed, 199 insertions(+), 238 deletions(-) diff --git a/.ruff-excludes.toml b/.ruff-excludes.toml index b28ccaa47d2..bfb3915e357 100644 --- a/.ruff-excludes.toml +++ b/.ruff-excludes.toml @@ -1250,15 +1250,6 @@ exclude = [ "./scripts/list_boards.py", "./scripts/list_hardware.py", "./scripts/list_shields.py", - "./scripts/logging/dictionary/database_gen.py", - "./scripts/logging/dictionary/dictionary_parser/data_types.py", - "./scripts/logging/dictionary/dictionary_parser/log_database.py", - "./scripts/logging/dictionary/dictionary_parser/log_parser.py", - "./scripts/logging/dictionary/dictionary_parser/log_parser_v1.py", - "./scripts/logging/dictionary/dictionary_parser/log_parser_v3.py", - "./scripts/logging/dictionary/dictionary_parser/utils.py", - "./scripts/logging/dictionary/log_parser.py", - "./scripts/logging/dictionary/log_parser_uart.py", "./scripts/make_bugs_pickle.py", "./scripts/net/enumerate_http_status.py", "./scripts/profiling/stackcollapse.py", @@ -1474,6 +1465,5 @@ exclude = [ "./tests/net/socket/tls_configurations/pytest/test_app_vs_openssl.py", "./tests/net/socket/udp/generate-c-string.py", "./tests/subsys/debug/gdbstub/pytest/test_gdbstub.py", - "./tests/subsys/logging/dictionary/pytest/test_logging_dictionary.py", "./tests/ztest/ztest_param/pytest/test_parameters.py", ] diff --git a/scripts/logging/dictionary/database_gen.py b/scripts/logging/dictionary/database_gen.py index 8d8a2cb1d0e..332d46cfe7c 100755 --- a/scripts/logging/dictionary/database_gen.py +++ b/scripts/logging/dictionary/database_gen.py @@ -43,9 +43,7 @@ STATIC_STRING_SECTIONS = [ ] # Sections that contains static strings but are not part of the binary (allocable). -REMOVED_STRING_SECTIONS = [ - 'log_strings' -] +REMOVED_STRING_SECTIONS = ['log_strings'] # Regulation expression to match DWARF location @@ -76,23 +74,18 @@ def parse_args(): argparser.add_argument("elffile", help="Zephyr ELF binary") argparser.add_argument("--build", help="Build ID") - argparser.add_argument("--build-header", - help="Header file containing BUILD_VERSION define") - argparser.add_argument("--debug", action="store_true", - help="Print extra debugging information") - argparser.add_argument("-v", "--verbose", action="store_true", - help="Print more information") + argparser.add_argument("--build-header", help="Header file containing BUILD_VERSION define") + argparser.add_argument("--debug", action="store_true", help="Print extra debugging information") + argparser.add_argument("-v", "--verbose", action="store_true", help="Print more information") outfile_grp = argparser.add_mutually_exclusive_group(required=True) - outfile_grp.add_argument("--json", - help="Output Dictionary Logging Database file in JSON") - outfile_grp.add_argument("--syst", - help="Output MIPI Sys-T Collateral XML file") + outfile_grp.add_argument("--json", help="Output Dictionary Logging Database file in JSON") + outfile_grp.add_argument("--syst", help="Output MIPI Sys-T Collateral XML file") return argparser.parse_args() -def extract_elf_code_data_sections(elf, wildcards = None): +def extract_elf_code_data_sections(elf, wildcards=None): """Find all sections in ELF file""" sections = {} @@ -101,17 +94,17 @@ def extract_elf_code_data_sections(elf, wildcards = None): # since they actually have code/data. # # On contrary, BSS is allocated but NOBITS. - if (((wildcards is not None) and (sect.name in wildcards)) or - ((sect['sh_flags'] & SH_FLAGS.SHF_ALLOC) == SH_FLAGS.SHF_ALLOC - and sect['sh_type'] == 'SHT_PROGBITS') + if ((wildcards is not None) and (sect.name in wildcards)) or ( + (sect['sh_flags'] & SH_FLAGS.SHF_ALLOC) == SH_FLAGS.SHF_ALLOC + and sect['sh_type'] == 'SHT_PROGBITS' ): sections[sect.name] = { - 'name' : sect.name, - 'size' : sect['sh_size'], - 'start' : sect['sh_addr'], - 'end' : sect['sh_addr'] + sect['sh_size'] - 1, - 'data' : sect.data(), - } + 'name': sect.name, + 'size': sect['sh_size'], + 'start': sect['sh_addr'], + 'end': sect['sh_addr'] + sect['sh_size'] - 1, + 'data': sect.data(), + } return sections @@ -121,11 +114,11 @@ def find_elf_sections(elf, sh_name): for section in elf.iter_sections(): if section.name == sh_name: ret = { - 'name' : section.name, - 'size' : section['sh_size'], - 'start' : section['sh_addr'], - 'end' : section['sh_addr'] + section['sh_size'] - 1, - 'data' : section.data(), + 'name': section.name, + 'size': section['sh_size'], + 'start': section['sh_addr'], + 'end': section['sh_addr'] + section['sh_size'] - 1, + 'data': section.data(), } return ret @@ -137,17 +130,20 @@ def get_kconfig_symbols(elf): """Get kconfig symbols from the ELF file""" for section in elf.iter_sections(): if isinstance(section, SymbolTableSection) and section['sh_type'] != 'SHT_DYNSYM': - return {sym.name: sym.entry.st_value - for sym in section.iter_symbols() - if sym.name.startswith("CONFIG_")} + return { + sym.name: sym.entry.st_value + for sym in section.iter_symbols() + if sym.name.startswith("CONFIG_") + } raise LookupError("Could not find symbol table") def find_log_const_symbols(elf): """Extract all "log_const_*" symbols from ELF file""" - symbol_tables = [s for s in elf.iter_sections() - if isinstance(s, elftools.elf.sections.SymbolTableSection)] + symbol_tables = [ + s for s in elf.iter_sections() if isinstance(s, elftools.elf.sections.SymbolTableSection) + ] ret_list = [] @@ -259,8 +255,7 @@ def process_kconfigs(elf, database): # # Use 32-bit timestamp? or 64-bit? if "CONFIG_LOG_TIMESTAMP_64BIT" in kconfigs: - database.add_kconfig("CONFIG_LOG_TIMESTAMP_64BIT", - kconfigs['CONFIG_LOG_TIMESTAMP_64BIT']) + database.add_kconfig("CONFIG_LOG_TIMESTAMP_64BIT", kconfigs['CONFIG_LOG_TIMESTAMP_64BIT']) def extract_logging_subsys_information(elf, database, string_mappings): @@ -289,9 +284,9 @@ def is_die_attr_ref(attr): """ Returns True if the DIE attribute is a reference. """ - return bool(attr.form in ('DW_FORM_ref1', 'DW_FORM_ref2', - 'DW_FORM_ref4', 'DW_FORM_ref8', - 'DW_FORM_ref')) + return bool( + attr.form in ('DW_FORM_ref1', 'DW_FORM_ref2', 'DW_FORM_ref4', 'DW_FORM_ref8', 'DW_FORM_ref') + ) def find_die_var_base_type(compile_unit, die, is_const): @@ -351,7 +346,8 @@ def extract_string_variables(elf): for die in compile_unit.iter_DIEs(): # Only care about variables with location information # and of type "char" - if die.tag == 'DW_TAG_variable' and ('DW_AT_type' in die.attributes + if die.tag == 'DW_TAG_variable' and ( + 'DW_AT_type' in die.attributes and 'DW_AT_location' in die.attributes and is_die_var_const_char(compile_unit, die) ): @@ -362,23 +358,25 @@ def extract_string_variables(elf): loc = loc_parser.parse_from_attribute(loc_attr, die.cu['version'], die) if isinstance(loc, LocationExpr): try: - addr = describe_DWARF_expr(loc.loc_expr, - dwarf_info.structs) + addr = describe_DWARF_expr(loc.loc_expr, dwarf_info.structs) matcher = DT_LOCATION_REGEX.match(addr) if matcher: addr = int(matcher.group(1), 16) if addr > 0: - strings.append({ - 'name': die.attributes['DW_AT_name'].value, - 'addr': addr, - 'die': die - }) + strings.append( + { + 'name': die.attributes['DW_AT_name'].value, + 'addr': addr, + 'die': die, + } + ) except KeyError: pass return strings + def try_decode_string(str_maybe): """Check if it is a printable string""" for encoding in STR_ENCODINGS: @@ -389,6 +387,7 @@ def try_decode_string(str_maybe): return None + def is_printable(b): # Check if string is printable according to Python # since the parser (written in Python) will need to @@ -398,6 +397,7 @@ def is_printable(b): # string.printable so they need to be checked separately. return (b in string.printable) or (b in ACCEPTABLE_ESCAPE_CHARS) + def extract_strings_in_one_section(section, str_mappings): """Extract NULL-terminated strings in one ELF section""" data = section['data'] @@ -412,7 +412,7 @@ def extract_strings_in_one_section(section, str_mappings): # End of possible string if start is not None: # Found potential string - str_maybe = data[start : idx] + str_maybe = data[start:idx] decoded_str = try_decode_string(str_maybe) if decoded_str is not None: @@ -425,8 +425,9 @@ def extract_strings_in_one_section(section, str_mappings): # (e.g. extended ASC-II characters) or control # characters (e.g. '\r' or '\n'), so simply print # the byte string instead. - logger.debug('Found string via extraction at ' + PTR_FMT + ': %s', - addr, str_maybe) + logger.debug( + 'Found string via extraction at ' + PTR_FMT + ': %s', addr, str_maybe + ) # GCC-based toolchain will reuse the NULL character # for empty strings. There is no way to know which @@ -435,8 +436,7 @@ def extract_strings_in_one_section(section, str_mappings): null_addr = section['start'] + idx str_mappings[null_addr] = '' - logger.debug('Found null string via extraction at ' + PTR_FMT, - null_addr) + logger.debug('Found null string via extraction at ' + PTR_FMT, null_addr) start = None else: # Non-printable byte, remove start location @@ -461,8 +461,9 @@ def extract_static_strings(elf, database, section_extraction=False): one_str = extract_one_string_in_section(sect, str_var['addr']) if one_str is not None: string_mappings[str_var['addr']] = one_str - logger.debug('Found string variable at ' + PTR_FMT + ': %s', - str_var['addr'], one_str) + logger.debug( + 'Found string variable at ' + PTR_FMT + ': %s', str_var['addr'], one_str + ) break if section_extraction: @@ -478,8 +479,7 @@ def extract_static_strings(elf, database, section_extraction=False): for sect_name in string_sections: if sect_name in elf_sections: - rawstr_map = extract_strings_in_one_section(elf_sections[sect_name], - rawstr_map) + rawstr_map = extract_strings_in_one_section(elf_sections[sect_name], rawstr_map) for one_str in rawstr_map: if one_str not in string_mappings: @@ -562,13 +562,11 @@ def main(): # Write database file if args.json and not LogDatabase.write_json_database(args.json, database): - logger.error("ERROR: Cannot open database file for write: %s, exiting...", - args.json) + logger.error("ERROR: Cannot open database file for write: %s, exiting...", args.json) sys.exit(1) if args.syst and not LogDatabase.write_syst_database(args.syst, database): - logger.error("ERROR: Cannot open database file for write: %s, exiting...", - args.syst) + logger.error("ERROR: Cannot open database file for write: %s, exiting...", args.syst) sys.exit(1) elffile.close() diff --git a/scripts/logging/dictionary/dictionary_parser/data_types.py b/scripts/logging/dictionary/dictionary_parser/data_types.py index e6ab71dc674..2729fad3cac 100644 --- a/scripts/logging/dictionary/dictionary_parser/data_types.py +++ b/scripts/logging/dictionary/dictionary_parser/data_types.py @@ -14,6 +14,7 @@ import struct class DataTypes: """Class regarding data types, their alignments and sizes""" + INT = 0 UINT = 1 LONG = 2 @@ -47,14 +48,13 @@ class DataTypes: self.add_data_type(self.DOUBLE, "d") self.add_data_type(self.LONG_DOUBLE, "d") - @staticmethod def get_stack_min_align(arch, is_tgt_64bit): ''' Correspond to the VA_STACK_ALIGN and VA_STACK_MIN_ALIGN in cbprintf_internal.h. Note that there might be some - variations that is obtained via actually running through - the log parser. + variations that is obtained via actually running through + the log parser. Return a tuple where the first element is stack alignment value. The second element is true if alignment needs to @@ -102,7 +102,6 @@ class DataTypes: return (stack_min_align, need_further_align) - @staticmethod def get_data_type_align(data_type, is_tgt_64bit): ''' @@ -121,7 +120,6 @@ class DataTypes: return align - def add_data_type(self, data_type, fmt): """Add one data type""" if self.database.is_tgt_little_endian(): @@ -158,31 +156,26 @@ class DataTypes: # 'stack_align' should correspond to VA_STACK_ALIGN # in cbprintf_internal.h stack_align, need_more_align = DataTypes.get_stack_min_align( - self.database.get_arch(), - self.database.is_tgt_64bit()) + self.database.get_arch(), self.database.is_tgt_64bit() + ) if need_more_align: - stack_align = DataTypes.get_data_type_align(data_type, - self.database.is_tgt_64bit()) + stack_align = DataTypes.get_data_type_align(data_type, self.database.is_tgt_64bit()) self.data_types[data_type]['stack_align'] = stack_align - def get_sizeof(self, data_type): """Get sizeof() of a data type""" return self.data_types[data_type]['sizeof'] - def get_alignment(self, data_type): """Get the alignment of a data type""" return self.data_types[data_type]['align'] - def get_stack_alignment(self, data_type): """Get the stack alignment of a data type""" return self.data_types[data_type]['stack_align'] - def get_formatter(self, data_type): """Get the formatter for a data type""" return self.data_types[data_type]['fmt'] diff --git a/scripts/logging/dictionary/dictionary_parser/log_database.py b/scripts/logging/dictionary/dictionary_parser/log_database.py index a4514784853..8f59db17ed8 100644 --- a/scripts/logging/dictionary/dictionary_parser/log_database.py +++ b/scripts/logging/dictionary/dictionary_parser/log_database.py @@ -16,37 +16,37 @@ from .mipi_syst import gen_syst_xml_file from .utils import extract_one_string_in_section, find_string_in_mappings ARCHS = { - "arc" : { + "arc": { "kconfig": "CONFIG_ARC", }, - "arm" : { + "arm": { "kconfig": "CONFIG_ARM", }, - "arm64" : { + "arm64": { "kconfig": "CONFIG_ARM64", }, - "mips" : { + "mips": { "kconfig": "CONFIG_MIPS", }, - "sparc" : { + "sparc": { "kconfig": "CONFIG_SPARC", }, - "x86" : { + "x86": { "kconfig": "CONFIG_X86", }, - "posix" : { + "posix": { "kconfig": "CONFIG_ARCH_POSIX", }, - "riscv32e" : { + "riscv32e": { "kconfig": "CONFIG_RISCV_ISA_RV32E", }, - "riscv" : { + "riscv": { "kconfig": "CONFIG_RISCV", }, - "rx" : { + "rx": { "kconfig": "CONFIG_RX", }, - "xtensa" : { + "xtensa": { "kconfig": "CONFIG_XTENSA", }, } @@ -54,6 +54,7 @@ ARCHS = { class LogDatabase: """Class of log database""" + # Update this if database format of dictionary based logging # has changed ZEPHYR_DICT_LOG_VER = 3 @@ -74,32 +75,26 @@ class LogDatabase: self.database = new_db - def get_version(self): """Get Database Version""" return self.database['version'] - def get_build_id(self): """Get Build ID""" return self.database['build_id'] - def set_build_id(self, build_id): """Set Build ID in Database""" self.database['build_id'] = build_id - def get_arch(self): """Get the Target Architecture""" return self.database['arch'] - def set_arch(self, arch): """Set the Target Architecture""" self.database['arch'] = arch - def get_tgt_bits(self): """Get Target Bitness: 32 or 64""" if 'bits' in self.database['target']: @@ -107,12 +102,10 @@ class LogDatabase: return None - def set_tgt_bits(self, bits): """Set Target Bitness: 32 or 64""" self.database['target']['bits'] = bits - def is_tgt_64bit(self): """Return True if target is 64-bit, False if 32-bit. None if error.""" @@ -127,7 +120,6 @@ class LogDatabase: return None - def get_tgt_endianness(self): """ Get Target Endianness. @@ -139,7 +131,6 @@ class LogDatabase: return None - def set_tgt_endianness(self, endianness): """ Set Target Endianness @@ -148,7 +139,6 @@ class LogDatabase: """ self.database['target']['little_endianness'] = endianness - def is_tgt_little_endian(self): """Return True if target is little endian""" if 'little_endianness' not in self.database['target']: @@ -156,22 +146,18 @@ class LogDatabase: return self.database['target']['little_endianness'] == self.LITTLE_ENDIAN - def get_string_mappings(self): """Get string mappings to database""" return self.database['string_mappings'] - def set_string_mappings(self, database): """Add string mappings to database""" self.database['string_mappings'] = database - def has_string_mappings(self): """Return True if there are string mappings in database""" return 'string_mappings' in self.database - def has_string_sections(self): """Return True if there are any static string sections""" if 'sections' not in self.database: @@ -179,7 +165,6 @@ class LogDatabase: return len(self.database['sections']) != 0 - def __find_string_in_mappings(self, string_ptr): """ Find string pointed by string_ptr in the string mapping @@ -187,7 +172,6 @@ class LogDatabase: """ return find_string_in_mappings(self.database['string_mappings'], string_ptr) - def __find_string_in_sections(self, string_ptr): """ Find string pointed by string_ptr in the binary data @@ -201,7 +185,6 @@ class LogDatabase: return None - def find_string(self, string_ptr): """Find string pointed by string_ptr in the database. Return None if not found.""" @@ -215,17 +198,15 @@ class LogDatabase: return one_str - def add_log_instance(self, source_id, name, level, address): """Add one log instance into database""" self.database['log_subsys']['log_instances'][source_id] = { - 'source_id' : source_id, - 'name' : name, - 'level' : level, - 'addr' : address, + 'source_id': source_id, + 'name': name, + 'level': level, + 'addr': address, } - def get_log_source_string(self, domain_id, source_id): """Get the source string based on source ID""" # JSON stores key as string, so we need to convert @@ -236,17 +217,14 @@ class LogDatabase: return f"unknown<{domain_id}:{source_id}>" - def add_kconfig(self, name, val): """Add a kconfig name-value pair into database""" self.database['kconfigs'][name] = val - def get_kconfigs(self): """Return kconfig name-value pairs""" return self.database['kconfigs'] - @staticmethod def read_json_database(db_file_name): """Read database from file and return a LogDatabase object""" @@ -277,7 +255,6 @@ class LogDatabase: return database - @staticmethod def write_json_database(db_file_name, database): """Write the database into file""" diff --git a/scripts/logging/dictionary/dictionary_parser/log_parser.py b/scripts/logging/dictionary/dictionary_parser/log_parser.py index ee37dda4f06..0e2b2580e35 100644 --- a/scripts/logging/dictionary/dictionary_parser/log_parser.py +++ b/scripts/logging/dictionary/dictionary_parser/log_parser.py @@ -20,9 +20,10 @@ LOG_LEVELS = [ ('err', Fore.RED), ('wrn', Fore.YELLOW), ('inf', Fore.GREEN), - ('dbg', Fore.BLUE) + ('dbg', Fore.BLUE), ] + def get_log_level_str_color(lvl): """Convert numeric log level to string""" if lvl < 0 or lvl >= len(LOG_LEVELS): @@ -56,12 +57,12 @@ def formalize_fmt_string(fmt_str): class LogParser(abc.ABC): """Abstract class of log parser""" + def __init__(self, database): self.database = database self.data_types = DataTypes(self.database) - @abc.abstractmethod def parse_log_data(self, logdata, debug=False): """Parse log data""" diff --git a/scripts/logging/dictionary/dictionary_parser/log_parser_v1.py b/scripts/logging/dictionary/dictionary_parser/log_parser_v1.py index 342fa5936e3..8279f2259df 100644 --- a/scripts/logging/dictionary/dictionary_parser/log_parser_v1.py +++ b/scripts/logging/dictionary/dictionary_parser/log_parser_v1.py @@ -62,6 +62,7 @@ logger = logging.getLogger("parser") class LogParserV1(LogParser): """Log Parser V1""" + def __init__(self, database): super().__init__(database=database) @@ -83,7 +84,6 @@ class LogParserV1(LogParser): else: self.fmt_msg_timestamp = endian + FMT_MSG_TIMESTAMP_32 - def __get_string(self, arg, arg_offset, string_tbl): one_str = self.database.find_string(arg) if one_str is not None: @@ -100,7 +100,6 @@ class LogParserV1(LogParser): return ret - def process_one_fmt_str(self, fmt_str, arg_list, string_tbl): """Parse the format string to extract arguments from the binary arglist and return a tuple usable with @@ -129,8 +128,7 @@ class LogParserV1(LogParser): elif fmt == '*': pass - elif fmt.isdecimal() or str.lower(fmt) == 'l' \ - or fmt in (' ', '#', '-', '+', '.', 'h'): + elif fmt.isdecimal() or str.lower(fmt) == 'l' or fmt in (' ', '#', '-', '+', '.', 'h'): # formatting modifiers, just ignore continue @@ -201,7 +199,6 @@ class LogParserV1(LogParser): return tuple(args) - @staticmethod def extract_string_table(str_tbl): """Extract string table in a packaged log message""" @@ -226,7 +223,6 @@ class LogParserV1(LogParser): return tbl - @staticmethod def print_hexdump(hex_data, prefix_len, color): """Print hex dump""" @@ -244,22 +240,25 @@ class LogParserV1(LogParser): chr_vals += " " elif chr_done == HEX_BYTES_IN_LINE: - print(f"{color}%s%s|%s{Fore.RESET}" % ((" " * prefix_len), - hex_vals, chr_vals)) + print(f"{color}%s%s|%s{Fore.RESET}" % ((" " * prefix_len), hex_vals, chr_vals)) hex_vals = "" chr_vals = "" chr_done = 0 if len(chr_vals) > 0: hex_padding = " " * (HEX_BYTES_IN_LINE - chr_done) - print(f"{color}%s%s%s|%s{Fore.RESET}" % ((" " * prefix_len), - hex_vals, hex_padding, chr_vals)) + print( + f"{color}%s%s%s|%s{Fore.RESET}" + % ((" " * prefix_len), hex_vals, hex_padding, chr_vals) + ) def get_full_msg_hdr_size(self): """Get the size of the full message header""" - return struct.calcsize(self.fmt_msg_type) + \ - struct.calcsize(self.fmt_msg_hdr) + \ - struct.calcsize(self.fmt_msg_timestamp) + return ( + struct.calcsize(self.fmt_msg_type) + + struct.calcsize(self.fmt_msg_hdr) + + struct.calcsize(self.fmt_msg_timestamp) + ) def get_normal_msg_size(self, logdata, offset): """Get the needed size of the normal log message at offset""" @@ -295,21 +294,21 @@ class LogParserV1(LogParser): offset_end_of_args += offset # Extra data after packaged log - extra_data = logdata[(offset + pkg_len):next_msg_offset] + extra_data = logdata[(offset + pkg_len) : next_msg_offset] # Number of appended strings in package - num_packed_strings = struct.unpack_from("B", logdata, offset+1)[0] + num_packed_strings = struct.unpack_from("B", logdata, offset + 1)[0] # Number of read-only string indexes - num_ro_str_indexes = struct.unpack_from("B", logdata, offset+2)[0] + num_ro_str_indexes = struct.unpack_from("B", logdata, offset + 2)[0] offset_end_of_args += num_ro_str_indexes # Number of read-write string indexes - num_rw_str_indexes = struct.unpack_from("B", logdata, offset+3)[0] + num_rw_str_indexes = struct.unpack_from("B", logdata, offset + 3)[0] offset_end_of_args += num_rw_str_indexes # Extract the string table in the packaged log message - string_tbl = self.extract_string_table(logdata[offset_end_of_args:(offset + pkg_len)]) + string_tbl = self.extract_string_table(logdata[offset_end_of_args : (offset + pkg_len)]) if len(string_tbl) != num_packed_strings: logger.error("------ Error extracting string table") @@ -324,11 +323,12 @@ class LogParserV1(LogParser): # the offset begins at 0 for va_list. However, the format string # itself is before the va_list, so need to go back the width of # a pointer. - fmt_str_ptr = struct.unpack_from(self.data_types.get_formatter(DataTypes.PTR), - logdata, offset)[0] - fmt_str = self.__get_string(fmt_str_ptr, - -self.data_types.get_sizeof(DataTypes.PTR), - string_tbl) + fmt_str_ptr = struct.unpack_from( + self.data_types.get_formatter(DataTypes.PTR), logdata, offset + )[0] + fmt_str = self.__get_string( + fmt_str_ptr, -self.data_types.get_sizeof(DataTypes.PTR), string_tbl + ) offset += self.data_types.get_sizeof(DataTypes.PTR) if not fmt_str: @@ -361,7 +361,6 @@ class LogParserV1(LogParser): msg_type = struct.unpack_from(self.fmt_msg_type, logdata, offset)[0] if msg_type == MSG_TYPE_DROPPED: - if offset + struct.calcsize(self.fmt_dropped_cnt) > len(logdata): return False, offset offset += struct.calcsize(self.fmt_msg_type) @@ -372,9 +371,9 @@ class LogParserV1(LogParser): print(f"--- {num_dropped} messages dropped ---") elif msg_type == MSG_TYPE_NORMAL: - - if ((offset + self.get_full_msg_hdr_size() > len(logdata)) or - (offset + self.get_normal_msg_size(logdata, offset) > len(logdata))): + if (offset + self.get_full_msg_hdr_size() > len(logdata)) or ( + offset + self.get_normal_msg_size(logdata, offset) > len(logdata) + ): return False, offset offset += struct.calcsize(self.fmt_msg_type) diff --git a/scripts/logging/dictionary/dictionary_parser/log_parser_v3.py b/scripts/logging/dictionary/dictionary_parser/log_parser_v3.py index 6bdf4e23e46..78d726a55c7 100644 --- a/scripts/logging/dictionary/dictionary_parser/log_parser_v3.py +++ b/scripts/logging/dictionary/dictionary_parser/log_parser_v3.py @@ -65,6 +65,7 @@ logger = logging.getLogger("parser") class LogParserV3(LogParser): """Log Parser V1""" + def __init__(self, database): super().__init__(database=database) @@ -88,7 +89,6 @@ class LogParserV3(LogParser): else: self.fmt_msg_timestamp = endian + FMT_MSG_TIMESTAMP_32 - def __get_string(self, arg, arg_offset, string_tbl): one_str = self.database.find_string(arg) if one_str is not None: @@ -105,7 +105,6 @@ class LogParserV3(LogParser): return ret - def process_one_fmt_str(self, fmt_str, arg_list, string_tbl): """Parse the format string to extract arguments from the binary arglist and return a tuple usable with @@ -134,8 +133,7 @@ class LogParserV3(LogParser): elif fmt == '*': pass - elif fmt.isdecimal() or str.lower(fmt) == 'l' \ - or fmt in (' ', '#', '-', '+', '.', 'h'): + elif fmt.isdecimal() or str.lower(fmt) == 'l' or fmt in (' ', '#', '-', '+', '.', 'h'): # formatting modifiers, just ignore continue @@ -206,7 +204,6 @@ class LogParserV3(LogParser): return tuple(args) - @staticmethod def extract_string_table(str_tbl): """Extract string table in a packaged log message""" @@ -231,7 +228,6 @@ class LogParserV3(LogParser): return tbl - @staticmethod def print_hexdump(hex_data, prefix_len, color): """Print hex dump""" @@ -249,22 +245,25 @@ class LogParserV3(LogParser): chr_vals += " " elif chr_done == HEX_BYTES_IN_LINE: - print(f"{color}%s%s|%s{Fore.RESET}" % ((" " * prefix_len), - hex_vals, chr_vals)) + print(f"{color}%s%s|%s{Fore.RESET}" % ((" " * prefix_len), hex_vals, chr_vals)) hex_vals = "" chr_vals = "" chr_done = 0 if len(chr_vals) > 0: hex_padding = " " * (HEX_BYTES_IN_LINE - chr_done) - print(f"{color}%s%s%s|%s{Fore.RESET}" % ((" " * prefix_len), - hex_vals, hex_padding, chr_vals)) + print( + f"{color}%s%s%s|%s{Fore.RESET}" + % ((" " * prefix_len), hex_vals, hex_padding, chr_vals) + ) def get_full_msg_hdr_size(self): """Get the size of the full message header""" - return struct.calcsize(self.fmt_msg_type) + \ - struct.calcsize(self.fmt_msg_hdr) + \ - struct.calcsize(self.fmt_msg_timestamp) + return ( + struct.calcsize(self.fmt_msg_type) + + struct.calcsize(self.fmt_msg_hdr) + + struct.calcsize(self.fmt_msg_timestamp) + ) def get_normal_msg_size(self, logdata, offset): """Get the needed size of the normal log message at offset""" @@ -278,8 +277,9 @@ class LogParserV3(LogParser): def parse_one_normal_msg(self, logdata, offset): """Parse one normal log message and print the encoded message""" # Parse log message header - domain_lvl, pkg_len, data_len, source_id = struct.unpack_from(self.fmt_msg_hdr, - logdata, offset) + domain_lvl, pkg_len, data_len, source_id = struct.unpack_from( + self.fmt_msg_hdr, logdata, offset + ) offset += struct.calcsize(self.fmt_msg_hdr) timestamp = struct.unpack_from(self.fmt_msg_timestamp, logdata, offset)[0] @@ -305,21 +305,21 @@ class LogParserV3(LogParser): offset_end_of_args += offset # Extra data after packaged log - extra_data = logdata[(offset + pkg_len):next_msg_offset] + extra_data = logdata[(offset + pkg_len) : next_msg_offset] # Number of appended strings in package - num_packed_strings = struct.unpack_from("B", logdata, offset+1)[0] + num_packed_strings = struct.unpack_from("B", logdata, offset + 1)[0] # Number of read-only string indexes - num_ro_str_indexes = struct.unpack_from("B", logdata, offset+2)[0] + num_ro_str_indexes = struct.unpack_from("B", logdata, offset + 2)[0] offset_end_of_args += num_ro_str_indexes # Number of read-write string indexes - num_rw_str_indexes = struct.unpack_from("B", logdata, offset+3)[0] + num_rw_str_indexes = struct.unpack_from("B", logdata, offset + 3)[0] offset_end_of_args += num_rw_str_indexes # Extract the string table in the packaged log message - string_tbl = self.extract_string_table(logdata[offset_end_of_args:(offset + pkg_len)]) + string_tbl = self.extract_string_table(logdata[offset_end_of_args : (offset + pkg_len)]) if len(string_tbl) != num_packed_strings: logger.error("------ Error extracting string table") @@ -334,11 +334,12 @@ class LogParserV3(LogParser): # the offset begins at 0 for va_list. However, the format string # itself is before the va_list, so need to go back the width of # a pointer. - fmt_str_ptr = struct.unpack_from(self.data_types.get_formatter(DataTypes.PTR), - logdata, offset)[0] - fmt_str = self.__get_string(fmt_str_ptr, - -self.data_types.get_sizeof(DataTypes.PTR), - string_tbl) + fmt_str_ptr = struct.unpack_from( + self.data_types.get_formatter(DataTypes.PTR), logdata, offset + )[0] + fmt_str = self.__get_string( + fmt_str_ptr, -self.data_types.get_sizeof(DataTypes.PTR), string_tbl + ) offset += self.data_types.get_sizeof(DataTypes.PTR) if not fmt_str: @@ -372,7 +373,6 @@ class LogParserV3(LogParser): msg_type = struct.unpack_from(self.fmt_msg_type, logdata, offset)[0] if msg_type == MSG_TYPE_DROPPED: - if offset + struct.calcsize(self.fmt_dropped_cnt) > len(logdata): return False, offset @@ -384,9 +384,9 @@ class LogParserV3(LogParser): print(f"--- {num_dropped} messages dropped ---") elif msg_type == MSG_TYPE_NORMAL: - - if ((offset + self.get_full_msg_hdr_size() > len(logdata)) or - (offset + self.get_normal_msg_size(logdata, offset) > len(logdata))): + if (offset + self.get_full_msg_hdr_size() > len(logdata)) or ( + offset + self.get_normal_msg_size(logdata, offset) > len(logdata) + ): return False, offset offset += struct.calcsize(self.fmt_msg_type) diff --git a/scripts/logging/dictionary/dictionary_parser/utils.py b/scripts/logging/dictionary/dictionary_parser/utils.py index c6ad4f2a9ff..f6a8e2dc052 100644 --- a/scripts/logging/dictionary/dictionary_parser/utils.py +++ b/scripts/logging/dictionary/dictionary_parser/utils.py @@ -62,6 +62,6 @@ def find_string_in_mappings(string_mappings, str_ptr): for ptr, string in string_mappings.items(): if ptr <= str_ptr < (ptr + len(string)): whole_str = string_mappings[ptr] - return whole_str[str_ptr - ptr:] + return whole_str[str_ptr - ptr :] return None diff --git a/scripts/logging/dictionary/log_parser.py b/scripts/logging/dictionary/log_parser.py index db0f0c51ca4..9ac92fc29c5 100755 --- a/scripts/logging/dictionary/log_parser.py +++ b/scripts/logging/dictionary/log_parser.py @@ -32,12 +32,13 @@ def parse_args(): argparser.add_argument("dbfile", help="Dictionary Logging Database file") argparser.add_argument("logfile", help="Log Data file") - argparser.add_argument("--hex", action="store_true", - help="Log Data file is in hexadecimal strings") - argparser.add_argument("--rawhex", action="store_true", - help="Log file only contains hexadecimal log data") - argparser.add_argument("--debug", action="store_true", - help="Print extra debugging information") + argparser.add_argument( + "--hex", action="store_true", help="Log Data file is in hexadecimal strings" + ) + argparser.add_argument( + "--rawhex", action="store_true", help="Log file only contains hexadecimal log data" + ) + argparser.add_argument("--debug", action="store_true", help="Print extra debugging information") return argparser.parse_args() @@ -135,5 +136,6 @@ def main(): ) sys.exit(1) + if __name__ == "__main__": main() diff --git a/scripts/logging/dictionary/log_parser_uart.py b/scripts/logging/dictionary/log_parser_uart.py index 1d4f6847c89..dc479da4623 100755 --- a/scripts/logging/dictionary/log_parser_uart.py +++ b/scripts/logging/dictionary/log_parser_uart.py @@ -25,8 +25,7 @@ def parse_args(): argparser.add_argument("dbfile", help="Dictionary Logging Database file") argparser.add_argument("serialPort", help="Port where the logs are generated") argparser.add_argument("baudrate", help="Serial Port baud rate") - argparser.add_argument("--debug", action="store_true", - help="Print extra debugging information") + argparser.add_argument("--debug", action="store_true", help="Print extra debugging information") return argparser.parse_args() @@ -34,8 +33,7 @@ def parse_args(): def main(): """function of serial parser""" - print("This script is deprecated. Use 'live_log_parser.py' instead.", - file=sys.stderr) + print("This script is deprecated. Use 'live_log_parser.py' instead.", file=sys.stderr) # Convert the arguments to the format expected by live_log_parser, and invoke it directly. args = parse_args() @@ -47,11 +45,7 @@ def main(): if args.debug: sys.argv.append('--debug') - sys.argv += [ - 'serial', - args.serialPort, - args.baudrate - ] + sys.argv += ['serial', args.serialPort, args.baudrate] live_log_parser.main() diff --git a/tests/subsys/logging/dictionary/pytest/test_logging_dictionary.py b/tests/subsys/logging/dictionary/pytest/test_logging_dictionary.py index f1586b56c99..fc95321e689 100644 --- a/tests/subsys/logging/dictionary/pytest/test_logging_dictionary.py +++ b/tests/subsys/logging/dictionary/pytest/test_logging_dictionary.py @@ -20,6 +20,7 @@ ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") logger = logging.getLogger(__name__) + def process_logs(dut: DeviceAdapter, build_dir): ''' This grabs the encoded log from console and parse the log @@ -41,7 +42,7 @@ def process_logs(dut: DeviceAdapter, build_dir): # as the log parser requires file as input. # Timeout is intentionally long. Twister will # timeout earlier with per-test timeout. - handler_output = dut.readlines_until(regex = '.*##ZLOGV1##[0-9]+', timeout = 600.0) + handler_output = dut.readlines_until(regex='.*##ZLOGV1##[0-9]+', timeout=600.0) # Join all the output lines together handler_output = ''.join(ho.strip() for ho in handler_output) @@ -73,45 +74,49 @@ def expected_regex_common(): the decoded log lines. ''' return [ - # *** Booting Zephyr OS build *** - re.compile(r'.*[*][*][*] Booting Zephyr OS build [0-9a-z.-]+'), - # Hello World! - re.compile(r'[\s]+Hello World! [\w-]+'), - # [ 10] hello_world: error string - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: error string'), - # [ 10] hello_world: main: debug string - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: debug string'), - # [ 10] hello_world: info string - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: info string'), - # [ 10] hello_world: main: int8_t 1, uint8_t 2 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int8_t 1, uint8_t 2'), - # [ 10] hello_world: main: int16_t 16, uint16_t 17 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int16_t 16, uint16_t 17'), - # [ 10] hello_world: main: int32_t 32, uint32_t 33 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int32_t 32, uint32_t 33'), - # [ 10] hello_world: main: int64_t 64, uint64_t 65 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int64_t 64, uint64_t 65'), - # [ 10] hello_world: main: char ! - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: char !'), - # [ 10] hello_world: main: s str static str c str - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: s str static str c str'), - # [ 10] hello_world: main: d str dynamic str - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: d str dynamic str'), - # [ 10] hello_world: main: mixed str dynamic str --- dynamic str \ - # --- another dynamic str --- another dynamic str - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: mixed str dynamic str ' - '--- dynamic str --- another dynamic str --- another dynamic str'), - # [ 10] hello_world: main: mixed c/s ! static str dynamic str static str ! - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: mixed c/s ! static str ' - 'dynamic str static str !'), - # [ 10] hello_world: main: pointer 0x1085f9 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: pointer 0x[0-9a-f]+'), - # [ 10] hello_world: main: For HeXdUmP! - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: For HeXdUmP!'), - # 48 45 58 44 55 4d 50 21 20 48 45 58 44 55 4d 50 |HEXDUMP! HEXDUMP - re.compile(r'[\s]+[ ]+[0-9a-f ]{48,52}[|]HEXDUMP! HEXDUMP'), - # 40 20 48 45 58 44 55 4d 50 23 |@ HEXDUM P# - re.compile(r'[\s]+[ ]+[0-9a-f ]{48,52}[|]@ HEXDUM P#'), + # *** Booting Zephyr OS build *** + re.compile(r'.*[*][*][*] Booting Zephyr OS build [0-9a-z.-]+'), + # Hello World! + re.compile(r'[\s]+Hello World! [\w-]+'), + # [ 10] hello_world: error string + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: error string'), + # [ 10] hello_world: main: debug string + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: debug string'), + # [ 10] hello_world: info string + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: info string'), + # [ 10] hello_world: main: int8_t 1, uint8_t 2 + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int8_t 1, uint8_t 2'), + # [ 10] hello_world: main: int16_t 16, uint16_t 17 + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int16_t 16, uint16_t 17'), + # [ 10] hello_world: main: int32_t 32, uint32_t 33 + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int32_t 32, uint32_t 33'), + # [ 10] hello_world: main: int64_t 64, uint64_t 65 + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: int64_t 64, uint64_t 65'), + # [ 10] hello_world: main: char ! + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: char !'), + # [ 10] hello_world: main: s str static str c str + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: s str static str c str'), + # [ 10] hello_world: main: d str dynamic str + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: d str dynamic str'), + # [ 10] hello_world: main: mixed str dynamic str --- dynamic str \ + # --- another dynamic str --- another dynamic str + re.compile( + r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: mixed str dynamic str ' + '--- dynamic str --- another dynamic str --- another dynamic str' + ), + # [ 10] hello_world: main: mixed c/s ! static str dynamic str static str ! + re.compile( + r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: mixed c/s ! static str ' + 'dynamic str static str !' + ), + # [ 10] hello_world: main: pointer 0x1085f9 + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: pointer 0x[0-9a-f]+'), + # [ 10] hello_world: main: For HeXdUmP! + re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: For HeXdUmP!'), + # 48 45 58 44 55 4d 50 21 20 48 45 58 44 55 4d 50 |HEXDUMP! HEXDUMP + re.compile(r'[\s]+[ ]+[0-9a-f ]{48,52}[|]HEXDUMP! HEXDUMP'), + # 40 20 48 45 58 44 55 4d 50 23 |@ HEXDUM P# + re.compile(r'[\s]+[ ]+[0-9a-f ]{48,52}[|]@ HEXDUM P#'), ] @@ -121,9 +126,11 @@ def expected_regex_fpu(): the decoded log lines for FPU builds. ''' return [ - # [ 10] hello_world: main: float 66.669998, double 68.690000 - re.compile(r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: ' - r'float 66[\.][0-9-\.]+, double 68[\.][0-9-\.]+'), + # [ 10] hello_world: main: float 66.669998, double 68.690000 + re.compile( + r'[\s]+[\[][0-9,:\. ]+[\]] hello_world: main: ' + r'float 66[\.][0-9-\.]+, double 68[\.][0-9-\.]+' + ), ]