michael/zephyr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

twister: refactor twister_main to simplify the system tests

Browse Source 
Refactored twister_main.py module to simplify the code and system tests.
Removed the need to patch `sys.argv` in blackbox tests.

Signed-off-by: Lukasz Fundakowski <lukasz.fundakowski@nordicsemi.no>
This commit is contained in:
Lukasz Fundakowski
2025-09-01 18:56:40 +02:00
committed by Benjamin Cabé
parent 654d794dd0
commit 4f637258ef
7 changed files with 78 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
scripts/pylib/twister/twisterlib/twister_main.py
View File

@@ -9,11 +9,17 @@ import os
import shutil
import sys
import time
from collections.abc import Sequence
import colorama
from colorama import Fore
from twisterlib.coverage import run_coverage
from twisterlib.environment import TwisterEnv
from twisterlib.environment import (
TwisterEnv,
add_parse_arguments,
parse_arguments,
python_version_guard,
)
from twisterlib.hardwaremap import HardwareMap
from twisterlib.log_helper import close_logging, setup_logging
from twisterlib.package import Artifacts
@@ -27,7 +33,7 @@ def init_color(colorama_strip):
colorama.init(strip=colorama_strip)
def twister(options: argparse.Namespace, default_options: argparse.Namespace):
def twister(options: argparse.Namespace, default_options: argparse.Namespace) -> int:
start_time = time.time()
# Configure color output
@@ -230,9 +236,17 @@ def twister(options: argparse.Namespace, default_options: argparse.Namespace):
return 0
def main(options: argparse.Namespace, default_options: argparse.Namespace):
def main(argv: Sequence[str] | None = None) -> int:
"""Main function to run twister."""
try:
return_code = twister(options, default_options)
python_version_guard()
parser = add_parse_arguments()
options = parse_arguments(parser, argv)
default_options = parse_arguments(parser, [], on_init=False)
return twister(options, default_options)
finally:
close_logging()
return return_code
if (os.name != "nt") and os.isatty(1):
# (OS is not Windows) and (stdout is interactive)
os.system("stty sane <&1")

55
scripts/tests/twister_blackbox/test_device.py
View File

@@ -6,7 +6,6 @@
Blackbox tests for twister's command line functions related to test filtering.
"""
import importlib
from unittest import mock
import os
import pytest
@@ -14,63 +13,35 @@ import sys
import re
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
from conftest import TEST_DATA, suite_filename_mock
from twisterlib.testplan import TestPlan
from twisterlib.twister_main import main as twister_main
class TestDevice:
TESTDATA_1 = [
(
1234,
),
(
4321,
),
(
1324,
)
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@pytest.mark.parametrize(
'seed',
TESTDATA_1,
ids=[
'seed 1234',
'seed 4321',
'seed 1324'
],
[1234, 4321, 1324],
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
def test_seed(self, capfd, out_path, seed):
test_platforms = ['native_sim']
path = os.path.join(TEST_DATA, 'tests', 'seed_native_sim')
args = ['--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',] + \
['--seed', f'{seed[0]}'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
args = [
'--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',
'--seed', f'{seed}',
*[val for pair in zip(['-p'] * len(test_platforms), test_platforms) for val in pair]
]
return_value = twister_main(args)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == '1'
assert return_value == 1
expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed[0])
assert re.search(expected_line, err)
expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed)
assert re.search(expected_line, err), f'Regex not found: r"{expected_line}"'

37
scripts/tests/twister_blackbox/test_disable.py
View File

@@ -6,7 +6,6 @@
Blackbox tests for twister's command line functions related to disable features.
"""
import importlib
import pytest
from unittest import mock
import os
@@ -14,8 +13,9 @@ import sys
import re
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
from conftest import TEST_DATA, suite_filename_mock
from twisterlib.testplan import TestPlan
from twisterlib.twister_main import main as twister_main
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
@@ -41,30 +41,17 @@ class TestDisable:
os.path.join(TEST_DATA, 'tests', 'always_warning'),
['qemu_x86'],
'--disable-warnings-as-errors',
'0'
0
),
(
os.path.join(TEST_DATA, 'tests', 'always_warning'),
['qemu_x86'],
'-v',
'1'
1
),
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@pytest.mark.parametrize(
'test_path, test_platforms, flag, expected, expected_none',
TESTDATA_1,
@@ -82,15 +69,14 @@ class TestDisable:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
return_value = twister_main(args)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == '0'
assert return_value == 0
if expected_none:
assert re.search(expected[0], err) is None, f"Not expected string in log: {expected[0]}"
assert re.search(expected[1], err) is None, f"Not expected: {expected[1]}"
@@ -117,13 +103,12 @@ class TestDisable:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
return_value = twister_main(args)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == expected_exit_code, \
f"Twister return not expected ({expected_exit_code}) exit code: ({sys_exit.value})"
assert return_value == expected_exit_code, \
f"Twister return not expected ({expected_exit_code}) exit code: ({return_value})"

28
scripts/tests/twister_blackbox/test_testlist.py
View File

@@ -6,29 +6,17 @@
Blackbox tests for twister's command line functions related to saving and loading a testlist.
"""
import importlib
from unittest import mock
import os
import pytest
import sys
import json
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock, clear_log_in_test
from conftest import TEST_DATA, suite_filename_mock, clear_log_in_test
from twisterlib.testplan import TestPlan
from twisterlib.twister_main import main as twister_main
class TestTestlist:
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
def test_save_tests(self, out_path):
@@ -42,11 +30,7 @@ class TestTestlist:
) for val in pair]
# Save agnostics tests
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
assert str(sys_exit.value) == '0'
assert twister_main(args) == 0
clear_log_in_test()
@@ -58,11 +42,7 @@ class TestTestlist:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
assert str(sys_exit.value) == '0'
assert twister_main(args) == 0
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)

60
scripts/tests/twister_blackbox/test_testplan.py
View File

@@ -6,22 +6,21 @@
Blackbox tests for twister's command line functions - those requiring testplan.json
"""
import importlib
from unittest import mock
import os
import pytest
import sys
import json
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
from conftest import TEST_DATA, suite_filename_mock
from twisterlib.testplan import TestPlan
from twisterlib.error import TwisterRuntimeError
from twisterlib.twister_main import main as twister_main
class TestTestPlan:
TESTDATA_1 = [
('dummy.agnostic.group2.a2_tests.assert1', SystemExit, 4),
('dummy.agnostic.group2.a2_tests.assert1', None, 4),
(
os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests',
'dummy', 'agnostic', 'group1', 'subgroup1',
@@ -39,17 +38,6 @@ class TestTestPlan:
(False, 7),
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@pytest.mark.parametrize(
'test, expected_exception, expected_subtest_count',
TESTDATA_1,
@@ -65,24 +53,21 @@ class TestTestPlan:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(expected_exception) as exc:
self.loader.exec_module(self.twister_module)
if expected_exception:
with pytest.raises(expected_exception):
twister_main(args)
else:
return_value = twister_main(args)
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)
filtered_j = [
(ts['platform'], ts['name'], tc['identifier']) \
for ts in j['testsuites'] \
for tc in ts['testcases'] if 'reason' not in tc
]
if expected_exception != SystemExit:
assert True
return
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)
filtered_j = [
(ts['platform'], ts['name'], tc['identifier']) \
for ts in j['testsuites'] \
for tc in ts['testcases'] if 'reason' not in tc
]
assert str(exc.value) == '0'
assert len(filtered_j) == expected_subtest_count
assert return_value == 0
assert len(filtered_j) == expected_subtest_count
@pytest.mark.parametrize(
'filter, expected_count',
@@ -98,11 +83,8 @@ class TestTestPlan:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as exc:
self.loader.exec_module(self.twister_module)
assert twister_main(args) == 0
assert str(exc.value) == '0'
import pprint
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)
@@ -132,11 +114,7 @@ class TestTestPlan:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as exc:
self.loader.exec_module(self.twister_module)
assert str(exc.value) == '0'
assert twister_main(args) == 0
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)

35
scripts/tests/twister_blackbox/test_tooling.py
View File

@@ -7,7 +7,6 @@ Blackbox tests for twister's command line functions related to Twister's tooling
"""
# pylint: disable=duplicate-code
import importlib
from unittest import mock
import os
import pytest
@@ -15,22 +14,13 @@ import sys
import json
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, suite_filename_mock
from conftest import TEST_DATA, sample_filename_mock, suite_filename_mock
from twisterlib.statuses import TwisterStatus
from twisterlib.testplan import TestPlan
from twisterlib.twister_main import main as twister_main
class TestTooling:
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@pytest.mark.parametrize(
'jobs',
@@ -47,15 +37,14 @@ class TestTooling:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
return_value = twister_main(args)
with open(os.path.join(out_path, 'twister.log')) as f:
log = f.read()
assert f'JOBS: {jobs}' in log
assert str(sys_exit.value) == '0'
assert return_value == 0
@mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock)
def test_force_toolchain(self, out_path):
@@ -69,11 +58,7 @@ class TestTooling:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
assert str(sys_exit.value) == '0'
return_value = twister_main(args)
with open(os.path.join(out_path, 'testplan.json')) as f:
j = json.load(f)
@@ -87,6 +72,8 @@ class TestTooling:
assert len(filtered_j) == 1
assert filtered_j[0][3] != TwisterStatus.FILTER
assert return_value == 0
@pytest.mark.parametrize(
'test_path, test_platforms',
[
@@ -110,12 +97,10 @@ class TestTooling:
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
return_value = twister_main(args)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == '0'
assert return_value == 0

20
scripts/twister
View File

@@ -186,7 +186,6 @@ import os
import sys
from pathlib import Path
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
if not ZEPHYR_BASE:
# This file has been zephyr/scripts/twister for years,
@@ -203,22 +202,7 @@ if not ZEPHYR_BASE:
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister/"))
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers"))
from twisterlib.environment import add_parse_arguments, parse_arguments, python_version_guard
from twisterlib.twister_main import main
from twisterlib.twister_main import main # noqa: E402
if __name__ == "__main__":
ret = 0
try:
python_version_guard()
parser = add_parse_arguments()
options = parse_arguments(parser, sys.argv[1:])
default_options = parse_arguments(parser, [], on_init=False)
ret = main(options, default_options)
finally:
if (os.name != "nt") and os.isatty(1):
# (OS is not Windows) and (stdout is interactive)
os.system("stty sane <&1")
sys.exit(ret)
sys.exit(main())