kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool is for checking the security hardening options of the Linux kernel.
   5
   6 Author: Alexander Popov <alex.popov@linux.com>
   7
   8 This module performs unit-testing of the kconfig-hardened-check engine.
   9 """
  10
  11 # pylint: disable=missing-function-docstring,line-too-long
  12
  13 import unittest
  14 import io
  15 import sys
  16 from collections import OrderedDict
  17 import json
  18 import inspect
  19 from .engine import KconfigCheck, CmdlineCheck, SysctlCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  20
  21
  22 class TestEngine(unittest.TestCase):
  23     """
  24     Example test scenario:
  25
  26         # 1. prepare the checklist
  27         config_checklist = []
  28         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  29         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  30         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'sysctl_name', 'expected_3')]
  31
  32         # 2. prepare the parsed kconfig options
  33         parsed_kconfig_options = OrderedDict()
  34         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  35
  36         # 3. prepare the parsed cmdline options
  37         parsed_cmdline_options = OrderedDict()
  38         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  39
  40         # 4. prepare the parsed sysctl options
  41         parsed_sysctl_options = OrderedDict()
  42         parsed_sysctl_options['sysctl_name'] = 'expected_3'
  43
  44         # 5. prepare the kernel version
  45         kernel_version = (42, 43)
  46
  47         # 6. run the engine
  48         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version)
  49
  50         # 7. check that the results are correct
  51         result = []
  52         self.get_engine_result(config_checklist, result, 'json')
  53         self.assertEqual(...
  54     """
  55
  56     @staticmethod
  57     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version):
  58         # populate the checklist with data
  59         if parsed_kconfig_options:
  60             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  61         if parsed_cmdline_options:
  62             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  63         if parsed_sysctl_options:
  64             populate_with_data(checklist, parsed_sysctl_options, 'sysctl')
  65         if kernel_version:
  66             populate_with_data(checklist, kernel_version, 'version')
  67
  68         # now everything is ready, perform the checks
  69         perform_checks(checklist)
  70
  71         # print the table with the results
  72         print(f'\n{inspect.stack()[1].function}():')
  73         print('=' * 121)
  74         for opt in checklist:
  75             opt.table_print('verbose', True) # verbose mode, with_results
  76             print()
  77             print('=' * 121)
  78
  79         # print the results in JSON
  80         result = []
  81         for opt in checklist:
  82             result.append(opt.json_dump(True)) # with_results
  83         print(json.dumps(result))
  84         print()
  85
  86     @staticmethod
  87     def get_engine_result(checklist, result, result_type):
  88         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  89                f'invalid result type "{result_type}"'
  90
  91         if result_type == 'json':
  92             for opt in checklist:
  93                 result.append(opt.json_dump(True)) # with_results
  94             return
  95
  96         captured_output = io.StringIO()
  97         stdout_backup = sys.stdout
  98         sys.stdout = captured_output
  99         for opt in checklist:
 100             if result_type == 'stdout_verbose':
 101                 opt.table_print('verbose', True) # verbose mode, with_results
 102             else:
 103                 opt.table_print(None, True) # normal mode, with_results
 104         sys.stdout = stdout_backup
 105         result.append(captured_output.getvalue())
 106
 107     def test_simple_kconfig(self):
 108         # 1. prepare the checklist
 109         config_checklist = []
 110         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 111         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 112         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 113         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 114         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 115         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 116         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 117         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 118         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 119         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 120
 121         # 2. prepare the parsed kconfig options
 122         parsed_kconfig_options = OrderedDict()
 123         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 124         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 125         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 126         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 127         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 128         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 129
 130         # 3. run the engine
 131         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 132
 133         # 4. check that the results are correct
 134         result = []
 135         self.get_engine_result(config_checklist, result, 'json')
 136         self.assertEqual(
 137                 result,
 138                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 139                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 140                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 141                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 142                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 143                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 144                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 145                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 146                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 147                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 148         )
 149
 150     def test_simple_cmdline(self):
 151         # 1. prepare the checklist
 152         config_checklist = []
 153         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 154         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 155         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 156         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 157         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 158         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 159         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 160         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 161         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 162         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 163
 164         # 2. prepare the parsed cmdline options
 165         parsed_cmdline_options = OrderedDict()
 166         parsed_cmdline_options['name_1'] = 'expected_1'
 167         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 168         parsed_cmdline_options['name_5'] = ''
 169         parsed_cmdline_options['name_7'] = ''
 170         parsed_cmdline_options['name_8'] = 'off'
 171         parsed_cmdline_options['name_9'] = '0'
 172
 173         # 3. run the engine
 174         self.run_engine(config_checklist, None, parsed_cmdline_options, None, None)
 175
 176         # 4. check that the results are correct
 177         result = []
 178         self.get_engine_result(config_checklist, result, 'json')
 179         self.assertEqual(
 180                 result,
 181                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 182                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 183                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 184                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 185                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 186                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 187                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 188                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 189                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 190                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 191         )
 192
 193     def test_simple_sysctl(self):
 194         # 1. prepare the checklist
 195         config_checklist = []
 196         config_checklist += [SysctlCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 197         config_checklist += [SysctlCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 198         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 199         config_checklist += [SysctlCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 200         config_checklist += [SysctlCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 201         config_checklist += [SysctlCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 202         config_checklist += [SysctlCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 203         config_checklist += [SysctlCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 204         config_checklist += [SysctlCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 205         config_checklist += [SysctlCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 206
 207         # 2. prepare the parsed sysctl options
 208         parsed_sysctl_options = OrderedDict()
 209         parsed_sysctl_options['name_1'] = 'expected_1'
 210         parsed_sysctl_options['name_2'] = 'UNexpected_2'
 211         parsed_sysctl_options['name_5'] = ''
 212         parsed_sysctl_options['name_7'] = ''
 213         parsed_sysctl_options['name_8'] = 'off'
 214         parsed_sysctl_options['name_9'] = '0'
 215
 216         # 3. run the engine
 217         self.run_engine(config_checklist, None, None, parsed_sysctl_options, None)
 218
 219         # 4. check that the results are correct
 220         result = []
 221         self.get_engine_result(config_checklist, result, 'json')
 222         self.assertEqual(
 223                 result,
 224                 [["name_1", "sysctl", "expected_1", "decision_1", "reason_1", "OK"],
 225                  ["name_2", "sysctl", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 226                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 227                  ["name_4", "sysctl", "is not set", "decision_4", "reason_4", "OK: is not found"],
 228                  ["name_5", "sysctl", "is present", "decision_5", "reason_5", "OK: is present"],
 229                  ["name_6", "sysctl", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 230                  ["name_7", "sysctl", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 231                  ["name_8", "sysctl", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 232                  ["name_9", "sysctl", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 233                  ["name_10", "sysctl", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 234         )
 235
 236     def test_complex_or(self):
 237         # 1. prepare the checklist
 238         config_checklist = []
 239         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 240                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 241         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 242                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 243         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 244                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 245         config_checklist += [OR(KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'expected_7'),
 246                                 KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not set'))]
 247         config_checklist += [OR(KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'expected_9'),
 248                                 KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is present'))]
 249         config_checklist += [OR(KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'expected_11'),
 250                                 KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'is not off'))]
 251
 252         # 2. prepare the parsed kconfig options
 253         parsed_kconfig_options = OrderedDict()
 254         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 255         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 256         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 257         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 258         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 259         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 260         parsed_kconfig_options['CONFIG_NAME_10'] = 'UNexpected_10'
 261         parsed_kconfig_options['CONFIG_NAME_12'] = 'really_not_off'
 262
 263         # 3. run the engine
 264         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 265
 266         # 4. check that the results are correct
 267         result = []
 268         self.get_engine_result(config_checklist, result, 'json')
 269         self.assertEqual(
 270                 result,
 271                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 272                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 273                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 274                  ["CONFIG_NAME_7", "kconfig", "expected_7", "decision_7", "reason_7", "OK: CONFIG_NAME_8 is not found"],
 275                  ["CONFIG_NAME_9", "kconfig", "expected_9", "decision_9", "reason_9", "OK: CONFIG_NAME_10 is present"],
 276                  ["CONFIG_NAME_11", "kconfig", "expected_11", "decision_11", "reason_11", "OK: CONFIG_NAME_12 is not off"]]
 277         )
 278
 279     def test_complex_and(self):
 280         # 1. prepare the checklist
 281         config_checklist = []
 282         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 283                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 284         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 285                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 286         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 287                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 288         config_checklist += [AND(KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'expected_7'),
 289                                  KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is present'))]
 290         config_checklist += [AND(KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'expected_9'),
 291                                  KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off'))]
 292         config_checklist += [AND(KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'expected_11'),
 293                                  KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'is not off'))]
 294
 295         # 2. prepare the parsed kconfig options
 296         parsed_kconfig_options = OrderedDict()
 297         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 298         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 299         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 300         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 301         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 302         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 303         parsed_kconfig_options['CONFIG_NAME_7'] = 'expected_7'
 304         parsed_kconfig_options['CONFIG_NAME_9'] = 'expected_9'
 305         parsed_kconfig_options['CONFIG_NAME_10'] = '0'
 306         parsed_kconfig_options['CONFIG_NAME_11'] = 'expected_11'
 307
 308         # 3. run the engine
 309         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 310
 311         # 4. check that the results are correct
 312         result = []
 313         self.get_engine_result(config_checklist, result, 'json')
 314         self.assertEqual(
 315                 result,
 316                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 317                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 318                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 319                  ["CONFIG_NAME_7", "kconfig", "expected_7", "decision_7", "reason_7", "FAIL: CONFIG_NAME_8 is not present"],
 320                  ["CONFIG_NAME_9", "kconfig", "expected_9", "decision_9", "reason_9", "FAIL: CONFIG_NAME_10 is off"],
 321                  ["CONFIG_NAME_11", "kconfig", "expected_11", "decision_11", "reason_11", "FAIL: CONFIG_NAME_12 is off, not found"]]
 322         )
 323
 324     def test_version(self):
 325         # 1. prepare the checklist
 326         config_checklist = []
 327         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 328                                 VersionCheck((41, 101)))]
 329         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 330                                 VersionCheck((44, 1)))]
 331         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 332                                 VersionCheck((42, 44)))]
 333         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 334                                 VersionCheck((42, 43)))]
 335
 336         # 2. prepare the parsed kconfig options
 337         parsed_kconfig_options = OrderedDict()
 338         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 339         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 340
 341         # 3. prepare the kernel version
 342         kernel_version = (42, 43)
 343
 344         # 4. run the engine
 345         self.run_engine(config_checklist, parsed_kconfig_options, None, None, kernel_version)
 346
 347         # 5. check that the results are correct
 348         result = []
 349         self.get_engine_result(config_checklist, result, 'json')
 350         self.assertEqual(
 351                 result,
 352                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 353                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 354                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 355                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 356         )
 357
 358     def test_stdout(self):
 359         # 1. prepare the checklist
 360         config_checklist = []
 361         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 362                                 CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 363                                 SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3'))]
 364         config_checklist += [AND(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 365                                  CmdlineCheck('reason_5', 'decision_5', 'name_5', 'expected_5'),
 366                                  SysctlCheck('reason_6', 'decision_6', 'name_6', 'expected_6'))]
 367
 368         # 2. prepare the parsed kconfig options
 369         parsed_kconfig_options = OrderedDict()
 370         parsed_kconfig_options['CONFIG_NAME_1'] = 'UNexpected_1'
 371
 372         # 3. prepare the parsed cmdline options
 373         parsed_cmdline_options = OrderedDict()
 374         parsed_cmdline_options['name_2'] = 'expected_2'
 375         parsed_cmdline_options['name_5'] = 'UNexpected_5'
 376
 377         # 4. prepare the parsed sysctl options
 378         parsed_sysctl_options = OrderedDict()
 379         parsed_sysctl_options['name_6'] = 'expected_6'
 380
 381         # 5. run the engine
 382         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 383
 384         # 6. check that the results are correct
 385         json_result = []
 386         self.get_engine_result(config_checklist, json_result, 'json')
 387         self.assertEqual(
 388                 json_result,
 389                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: name_2 is \"expected_2\""],
 390                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "FAIL: name_5 is not \"expected_5\""]]
 391         )
 392
 393         stdout_result = []
 394         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 395         self.assertEqual(
 396                 stdout_result,
 397                 [
 398 "\
 399 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | \x1b[32mOK: name_2 is \"expected_2\"\x1b[0m\
 400 CONFIG_NAME_4                           |kconfig| expected_4 |decision_4|     reason_4     | \x1b[31mFAIL: name_5 is not \"expected_5\"\x1b[0m\
 401 "               ]
 402         )
 403
 404         stdout_result = []
 405         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 406         self.assertEqual(
 407                 stdout_result,
 408                 [
 409 "\
 410     <<< OR >>>                                                                             | \x1b[32mOK: name_2 is \"expected_2\"\x1b[0m\n\
 411 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | \x1b[31mFAIL: \"UNexpected_1\"\x1b[0m\n\
 412 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | \x1b[32mOK\x1b[0m\n\
 413 name_3                                  |sysctl | expected_3 |decision_3|     reason_3     | None\
 414 "\
 415 "\
 416     <<< AND >>>                                                                            | \x1b[31mFAIL: name_5 is not \"expected_5\"\x1b[0m\n\
 417 CONFIG_NAME_4                           |kconfig| expected_4 |decision_4|     reason_4     | None\n\
 418 name_5                                  |cmdline| expected_5 |decision_5|     reason_5     | \x1b[31mFAIL: \"UNexpected_5\"\x1b[0m\n\
 419 name_6                                  |sysctl | expected_6 |decision_6|     reason_6     | \x1b[32mOK\x1b[0m\
 420 "               ]
 421         )
 422
 423     def test_value_overriding(self):
 424         # 1. prepare the checklist
 425         config_checklist = []
 426         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 427         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 428         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 429
 430         # 2. prepare the parsed kconfig options
 431         parsed_kconfig_options = OrderedDict()
 432         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 433
 434         # 3. prepare the parsed cmdline options
 435         parsed_cmdline_options = OrderedDict()
 436         parsed_cmdline_options['name_2'] = 'expected_2_new'
 437
 438         # 4. prepare the parsed sysctl options
 439         parsed_sysctl_options = OrderedDict()
 440         parsed_sysctl_options['name_3'] = 'expected_3_new'
 441
 442         # 5. run the engine
 443         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 444
 445         # 6. check that the results are correct
 446         result = []
 447         self.get_engine_result(config_checklist, result, 'json')
 448         self.assertEqual(
 449                 result,
 450                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 451                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 452                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 453         )
 454
 455         # 7. override expected value and perform the checks again
 456         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 457         perform_checks(config_checklist)
 458
 459         # 8. check that the results are correct
 460         result = []
 461         self.get_engine_result(config_checklist, result, 'json')
 462         self.assertEqual(
 463                 result,
 464                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 465                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 466                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 467         )
 468
 469         # 9. override expected value and perform the checks again
 470         override_expected_value(config_checklist, "name_2", "expected_2_new")
 471         perform_checks(config_checklist)
 472
 473         # 10. check that the results are correct
 474         result = []
 475         self.get_engine_result(config_checklist, result, 'json')
 476         self.assertEqual(
 477                 result,
 478                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 479                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 480                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 481         )
 482
 483         # 11. override expected value and perform the checks again
 484         override_expected_value(config_checklist, "name_3", "expected_3_new")
 485         perform_checks(config_checklist)
 486
 487         # 12. check that the results are correct
 488         result = []
 489         self.get_engine_result(config_checklist, result, 'json')
 490         self.assertEqual(
 491                 result,
 492                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 493                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 494                  ["name_3", "sysctl", "expected_3_new", "decision_3", "reason_3", "OK"]]
 495         )