build new dataset from error logs

scenarionet/verifier/utils.py

@@ -1,8 +1,12 @@
-import json
 import logging
 import multiprocessing
 import os
 
+from metadrive.scenario.scenario_description import ScenarioDescription as SD
+
+from scenarionet.verifier.error import ErrorDescription as ED
+from scenarionet.verifier.error import ErrorFile as EF
+
 logger = logging.getLogger(__name__)
 import tqdm
 from metadrive.envs.scenario_env import ScenarioEnv
@@ -12,15 +16,14 @@ from functools import partial
 
 
 def verify_loading_into_metadrive(dataset_path, result_save_dir, steps_to_run=1000, num_workers=8):
-    if result_save_dir is not None:
-        assert os.path.exists(result_save_dir
-                              ) and os.path.isdir(result_save_dir), "Argument result_save_dir must be an existing dir"
+    assert os.path.exists(result_save_dir) and os.path.isdir(result_save_dir), "result_save_dir must be an existing dir"
     num_scenario = get_number_of_scenarios(dataset_path)
     if num_scenario < num_workers:
         # single process
         logger.info("Use one worker, as num_scenario < num_workers:")
         num_workers = 1
 
+    # prepare arguments
     argument_list = []
     func = partial(loading_wrapper, dataset_path=dataset_path, steps_to_run=steps_to_run)
 
@@ -32,47 +35,46 @@ def verify_loading_into_metadrive(dataset_path, result_save_dir, steps_to_run=10
             scenario_num = num_scenario_each_worker
         argument_list.append([i * num_scenario_each_worker, scenario_num])
 
+    # Run, workers and process result from worker
     with multiprocessing.Pool(num_workers) as p:
         all_result = list(p.imap(func, argument_list))
     result = all([i[0] for i in all_result])
-    logs = []
-    for _, log in all_result:
-        logs += log
+    errors = []
+    for _, error in all_result:
+        errors += error
 
-    if result_save_dir is not None:
-        file_name = "error_scenarios_for_{}.json".format(os.path.basename(dataset_path))
-        with open(os.path.join(result_save_dir, file_name), "w+") as f:
-            json.dump(logs, f, indent=4)
+    # save result
+    EF.dump(result_save_dir, errors, dataset_path)
 
+    # logging
     if result:
         logger.info("All scenarios can be loaded successfully!")
     else:
         logger.info(
-            "Fail to load all scenarios, see log for more details! Number of failed scenarios: {}".format(len(logs)))
-    return result, logs
+            "Fail to load all scenarios, see log for more details! Number of failed scenarios: {}".format(len(errors)))
+    return result, errors
 
 
-def loading_into_metadrive(start_scenario_index, num_scenario, dataset_path, steps_to_run):
+def loading_into_metadrive(start_scenario_index, num_scenario, dataset_path, steps_to_run, metadrive_config=None):
     logger.info(
         "================ Begin Scenario Loading Verification for scenario {}-{} ================ \n".format(
             start_scenario_index, num_scenario + start_scenario_index))
     success = True
-    env = ScenarioEnv(
-        {
-            "agent_policy": ReplayEgoCarPolicy,
-            "num_scenarios": num_scenario,
-            "horizon": 1000,
-            "start_scenario_index": start_scenario_index,
-            "no_static_vehicles": False,
-            "data_directory": dataset_path,
-        }
-    )
+    metadrive_config = metadrive_config or {}
+    metadrive_config.update({
+        "agent_policy": ReplayEgoCarPolicy,
+        "num_scenarios": num_scenario,
+        "horizon": 1000,
+        "start_scenario_index": start_scenario_index,
+        "no_static_vehicles": False,
+        "data_directory": dataset_path,
+    })
+    env = ScenarioEnv(metadrive_config)
     logging.disable(logging.INFO)
-    error_files = []
-    try:
-        for scenario_index in tqdm.tqdm(range(start_scenario_index, start_scenario_index + num_scenario),
-                                        desc="Scenarios: {}-{}".format(start_scenario_index,
-                                                                       start_scenario_index + num_scenario)):
+    error_msgs = []
+    desc = "Scenarios: {}-{}".format(start_scenario_index, start_scenario_index + num_scenario)
+    for scenario_index in tqdm.tqdm(range(start_scenario_index, start_scenario_index + num_scenario), desc=desc):
+        try:
             env.reset(force_seed=scenario_index)
             arrive = False
             for _ in range(steps_to_run):
@@ -80,19 +82,17 @@ def loading_into_metadrive(start_scenario_index, num_scenario, dataset_path, ste
                 if d and info["arrive_dest"]:
                     arrive = True
             assert arrive, "Can not arrive destination"
-    except Exception as e:
-        file_name = env.engine.data_manager.summary_lookup[scenario_index]
-        file_path = os.path.join(dataset_path, env.engine.data_manager.mapping[file_name], file_name)
-        error_file = {"scenario_index": scenario_index, "file_path": file_path, "error": str(e)}
-        error_files.append(error_file)
-        logger.warning(
-            "\n Scenario Error, "
-            "scenario_index: {}, file_path: {}.\n Error message: {}".format(scenario_index, file_path, str(e))
-        )
-        success = False
-    finally:
-        env.close()
-    return success, error_files
+        except Exception as e:
+            file_name = env.engine.data_manager.summary_lookup[scenario_index]
+            file_path = os.path.join(dataset_path, env.engine.data_manager.mapping[file_name], file_name)
+            error_msg = ED.make(scenario_index, file_path, file_name, str(e))
+            error_msgs.append(error_msg)
+            success = False
+            # proceed to next scenario
+            continue
+
+    env.close()
+    return success, error_msgs
 
 
 def loading_wrapper(arglist, dataset_path, steps_to_run):