kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool is for checking the security hardening options of the Linux kernel.
   5
   6 Author: Alexander Popov <alex.popov@linux.com>
   7
   8 This module performs unit-testing of the kconfig-hardened-check engine.
   9 """
  10
  11 # pylint: disable=missing-function-docstring,line-too-long
  12
  13 import unittest
  14 import io
  15 import sys
  16 from collections import OrderedDict
  17 import json
  18 from .engine import KconfigCheck, CmdlineCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  19
  20
  21 class TestEngine(unittest.TestCase):
  22     """
  23     Example test scenario:
  24
  25         # 1. prepare the checklist
  26         config_checklist = []
  27         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  28         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  29
  30         # 2. prepare the parsed kconfig options
  31         parsed_kconfig_options = OrderedDict()
  32         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  33
  34         # 3. prepare the parsed cmdline options
  35         parsed_cmdline_options = OrderedDict()
  36         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  37
  38         # 4. prepare the kernel version
  39         kernel_version = (42, 43)
  40
  41         # 5. run the engine
  42         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version)
  43
  44         # 6. check that the results are correct
  45         result = []
  46         self.get_engine_result(config_checklist, result, 'json')
  47         self.assertEqual(...
  48     """
  49
  50     @staticmethod
  51     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version):
  52         # populate the checklist with data
  53         if parsed_kconfig_options:
  54             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  55         if parsed_cmdline_options:
  56             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  57         if kernel_version:
  58             populate_with_data(checklist, kernel_version, 'version')
  59
  60         # now everything is ready, perform the checks
  61         perform_checks(checklist)
  62
  63         # print the table with the results
  64         print('TABLE:')
  65         for opt in checklist:
  66             opt.table_print('verbose', True) # verbose mode, with_results
  67             print()
  68             print('=' * 121)
  69
  70         # print the results in JSON
  71         print('JSON:')
  72         result = []
  73         for opt in checklist:
  74             result.append(opt.json_dump(True)) # with_results
  75         print(json.dumps(result))
  76         print()
  77
  78     @staticmethod
  79     def get_engine_result(checklist, result, result_type):
  80         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  81                f'invalid result type "{result_type}"'
  82
  83         if result_type == 'json':
  84             for opt in checklist:
  85                 result.append(opt.json_dump(True)) # with_results
  86             return
  87
  88         captured_output = io.StringIO()
  89         stdout_backup = sys.stdout
  90         sys.stdout = captured_output
  91         for opt in checklist:
  92             if result_type == 'stdout_verbose':
  93                 opt.table_print('verbose', True) # verbose mode, with_results
  94             else:
  95                 opt.table_print(None, True) # normal mode, with_results
  96         sys.stdout = stdout_backup
  97         result.append(captured_output.getvalue())
  98
  99     def test_simple_kconfig(self):
 100         # 1. prepare the checklist
 101         config_checklist = []
 102         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 103         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 104         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 105         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 106         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 107         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 108         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 109         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 110         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 111         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 112
 113         # 2. prepare the parsed kconfig options
 114         parsed_kconfig_options = OrderedDict()
 115         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 116         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 117         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 118         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 119         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 120         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 121
 122         # 3. run the engine
 123         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 124
 125         # 4. check that the results are correct
 126         result = []
 127         self.get_engine_result(config_checklist, result, 'json')
 128         self.assertEqual(
 129                 result,
 130                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 131                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 132                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 133                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 134                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 135                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 136                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 137                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 138                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 139                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 140         )
 141
 142     def test_simple_cmdline(self):
 143         # 1. prepare the checklist
 144         config_checklist = []
 145         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 146         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 147         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 148         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 149         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 150         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 151         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 152         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 153         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 154         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 155
 156         # 2. prepare the parsed cmdline options
 157         parsed_cmdline_options = OrderedDict()
 158         parsed_cmdline_options['name_1'] = 'expected_1'
 159         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 160         parsed_cmdline_options['name_5'] = ''
 161         parsed_cmdline_options['name_7'] = ''
 162         parsed_cmdline_options['name_8'] = 'off'
 163         parsed_cmdline_options['name_9'] = '0'
 164
 165         # 3. run the engine
 166         self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 167
 168         # 4. check that the results are correct
 169         result = []
 170         self.get_engine_result(config_checklist, result, 'json')
 171         self.assertEqual(
 172                 result,
 173                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 174                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 175                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 176                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 177                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 178                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 179                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 180                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 181                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 182                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 183         )
 184
 185     def test_complex_or(self):
 186         # 1. prepare the checklist
 187         config_checklist = []
 188         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 189                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 190         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 191                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 192         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 193                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 194         config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
 195                                 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
 196         config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 197                                 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 198         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 199                                 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 200
 201         # 2. prepare the parsed kconfig options
 202         parsed_kconfig_options = OrderedDict()
 203         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 204         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 205         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 206         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 207         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 208         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 209         parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
 210         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 211
 212         # 3. run the engine
 213         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 214
 215         # 4. check that the results are correct
 216         result = []
 217         self.get_engine_result(config_checklist, result, 'json')
 218         self.assertEqual(
 219                 result,
 220                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 221                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 222                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 223                  ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
 224                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
 225                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
 226         )
 227
 228     def test_complex_and(self):
 229         # 1. prepare the checklist
 230         config_checklist = []
 231         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 232                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 233         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 234                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 235         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 236                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 237         config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 238                                  KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 239         config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 240                                  KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 241         config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
 242                                  KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
 243
 244         # 2. prepare the parsed kconfig options
 245         parsed_kconfig_options = OrderedDict()
 246         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 247         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 248         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 249         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 250         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 251         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 252         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 253         parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
 254         parsed_kconfig_options['CONFIG_NAME_11'] = '0'
 255         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 256
 257         # 3. run the engine
 258         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 259
 260         # 4. check that the results are correct
 261         result = []
 262         self.get_engine_result(config_checklist, result, 'json')
 263         self.assertEqual(
 264                 result,
 265                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 266                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 267                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 268                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
 269                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
 270                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
 271         )
 272
 273     def test_version(self):
 274         # 1. prepare the checklist
 275         config_checklist = []
 276         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 277                                 VersionCheck((41, 101)))]
 278         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 279                                 VersionCheck((44, 1)))]
 280         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 281                                 VersionCheck((42, 44)))]
 282         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 283                                 VersionCheck((42, 43)))]
 284
 285         # 2. prepare the parsed kconfig options
 286         parsed_kconfig_options = OrderedDict()
 287         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 288         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 289
 290         # 3. prepare the kernel version
 291         kernel_version = (42, 43)
 292
 293         # 4. run the engine
 294         self.run_engine(config_checklist, parsed_kconfig_options, None, kernel_version)
 295
 296         # 5. check that the results are correct
 297         result = []
 298         self.get_engine_result(config_checklist, result, 'json')
 299         self.assertEqual(
 300                 result,
 301                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 302                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 303                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 304                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 305         )
 306
 307     def test_stdout(self):
 308         # 1. prepare the checklist
 309         config_checklist = []
 310         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 311                                 AND(CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 312                                     KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')))]
 313         config_checklist += [AND(CmdlineCheck('reason_4', 'decision_4', 'name_4', 'expected_4'),
 314                                  OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 315                                     CmdlineCheck('reason_6', 'decision_6', 'name_6', 'expected_6')))]
 316
 317         # 2. prepare the parsed cmdline options
 318         parsed_cmdline_options = OrderedDict()
 319         parsed_cmdline_options['name_4'] = 'expected_4'
 320         parsed_cmdline_options['name_6'] = 'UNexpected_6'
 321
 322         # 3. run the engine
 323         self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 324
 325         # 4. check that the results are correct
 326         json_result = []
 327         self.get_engine_result(config_checklist, json_result, 'json')
 328         self.assertEqual(
 329                 json_result,
 330                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: is not found"],
 331                  ["name_4", "cmdline", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""]]
 332         )
 333
 334         stdout_result = []
 335         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 336         self.assertEqual(
 337                 stdout_result,
 338                 [
 339 "\
 340 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\
 341 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | FAIL: CONFIG_NAME_5 is not \"expected_5\"\
 342 "               ]
 343         )
 344
 345         stdout_result = []
 346         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 347         self.assertEqual(
 348                 stdout_result,
 349                 [
 350 "\
 351     <<< OR >>>                                                                             | FAIL: is not found\n\
 352 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\n\
 353     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_3 is not \"expected_3\"\n\
 354 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | None\n\
 355 CONFIG_NAME_3                           |kconfig| expected_3 |decision_3|     reason_3     | FAIL: is not found\
 356 "\
 357 "\
 358     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_5 is not \"expected_5\"\n\
 359 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | None\n\
 360     <<< OR >>>                                                                             | FAIL: is not found\n\
 361 CONFIG_NAME_5                           |kconfig| expected_5 |decision_5|     reason_5     | FAIL: is not found\n\
 362 name_6                                  |cmdline| expected_6 |decision_6|     reason_6     | FAIL: \"UNexpected_6\"\
 363 "               ]
 364         )
 365
 366     def test_value_overriding(self):
 367         # 1. prepare the checklist
 368         config_checklist = []
 369         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 370         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 371
 372         # 2. prepare the parsed kconfig options
 373         parsed_kconfig_options = OrderedDict()
 374         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 375
 376         # 3. prepare the parsed cmdline options
 377         parsed_cmdline_options = OrderedDict()
 378         parsed_cmdline_options['name_2'] = 'expected_2_new'
 379
 380         # 4. run the engine
 381         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, None)
 382
 383         # 5. check that the results are correct
 384         result = []
 385         self.get_engine_result(config_checklist, result, 'json')
 386         self.assertEqual(
 387                 result,
 388                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 389                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
 390         )
 391
 392         # 6. override expected value and perform the checks again
 393         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 394         perform_checks(config_checklist)
 395
 396         # 7. check that the results are correct
 397         result = []
 398         self.get_engine_result(config_checklist, result, 'json')
 399         self.assertEqual(
 400                 result,
 401                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 402                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
 403         )
 404
 405         # 8. override expected value and perform the checks again
 406         override_expected_value(config_checklist, "name_2", "expected_2_new")
 407         perform_checks(config_checklist)
 408
 409         # 9. check that the results are correct
 410         result = []
 411         self.get_engine_result(config_checklist, result, 'json')
 412         self.assertEqual(
 413                 result,
 414                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 415                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"]]
 416         )