fix combine bug

scenarionet/builder/utils.py

@@ -81,7 +81,11 @@ def combine_multiple_dataset(
         else:
             with open(osp.join(abs_dir_path, ScenarioDescription.DATASET.MAPPING_FILE), "rb+") as f:
                 mapping = pickle.load(f)
-        new_mapping = {k: os.path.relpath(abs_dir_path, output_abs_path) for k, v in mapping.items()}
+        new_mapping = {}
+        for file, rel_path in mapping.items():
+            # mapping to real file path
+            new_mapping[file] = os.path.relpath(osp.join(abs_dir_path, rel_path), output_abs_path)
+
         mappings.update(new_mapping)
 
     # apply filter stage
@@ -98,5 +102,3 @@ def combine_multiple_dataset(
     save_summary_anda_mapping(summary_file, mapping_file, summaries, mappings)
 
     return summaries, mappings
-
-

scenarionet/tests/local_test/_test_convert_nuplan.py

@@ -1,39 +0,0 @@
-"""
-This script aims to convert nuplan scenarios to ScenarioDescription, so that we can load any nuplan scenarios into
-MetaDrive.
-"""
-import argparse
-import os
-
-from scenarionet import SCENARIONET_DATASET_PATH
-from scenarionet.converter.nuplan.utils import get_nuplan_scenarios, convert_nuplan_scenario
-from scenarionet.converter.utils import write_to_directory
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--dataset_name", "-n", default="nuplan",
-                        help="Dataset name, will be used to generate scenario files")
-    parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "nuplan"),
-                        help="The path of the dataset")
-    parser.add_argument("--version", "-v", default='v1.1', help="version")
-    parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it")
-    args = parser.parse_args()
-
-    force_overwrite = True
-    dataset_name = args.dataset_name
-    output_path = args.dataset_path
-    version = args.version
-
-    data_root = os.path.join(os.getenv("NUPLAN_DATA_ROOT"), "nuplan-v1.1/splits/mini")
-    map_root = os.getenv("NUPLAN_MAPS_ROOT")
-    scenarios = get_nuplan_scenarios(data_root, map_root, logs=["2021.07.16.20.45.29_veh-35_01095_01486"])
-
-    write_to_directory(
-        convert_func=convert_nuplan_scenario,
-        scenarios=scenarios,
-        output_path=output_path,
-        dataset_version=version,
-        dataset_name=dataset_name,
-        force_overwrite=force_overwrite,
-        num_workers=8,
-    )

scenarionet/tests/local_test/_test_convert_nuscenes.py

@@ -1,39 +0,0 @@
-"""
-This script aims to convert nuscenes scenarios to ScenarioDescription, so that we can load any nuscenes scenarios into
-MetaDrive.
-"""
-import argparse
-import os.path
-
-from scenarionet import SCENARIONET_DATASET_PATH
-from scenarionet.converter.nuscenes.utils import convert_nuscenes_scenario, get_nuscenes_scenarios
-from scenarionet.converter.utils import write_to_directory
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--dataset_name", "-n", default="nuscenes",
-                        help="Dataset name, will be used to generate scenario files")
-    parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "nuscenes"),
-                        help="The path of the dataset")
-    parser.add_argument("--version", "-v", default='v1.0-mini',  help="version")
-    parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it")
-    args = parser.parse_args()
-
-    force_overwrite = True
-    dataset_name = args.dataset_name
-    output_path = args.dataset_path
-    version = args.version
-
-    dataroot = '/home/shady/data/nuscenes'
-    scenarios, nusc = get_nuscenes_scenarios(dataroot, version)
-
-    write_to_directory(
-        convert_func=convert_nuscenes_scenario,
-        scenarios=scenarios,
-        output_path=output_path,
-        dataset_version=version,
-        dataset_name=dataset_name,
-        force_overwrite=force_overwrite,
-        nuscenes=nusc,
-        num_workers=4
-    )

scenarionet/tests/local_test/_test_convert_pg.py

@@ -1,37 +0,0 @@
-# from metadrive.policy.expert_policy import ExpertPolicy
-import argparse
-import os.path
-
-import metadrive
-from metadrive.policy.idm_policy import IDMPolicy
-
-from scenarionet import SCENARIONET_DATASET_PATH
-from scenarionet.converter.pg.utils import get_pg_scenarios, convert_pg_scenario
-from scenarionet.converter.utils import write_to_directory
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--dataset_name", "-n", default="pg",
-                        help="Dataset name, will be used to generate scenario files")
-    parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "pg"),
-                        help="The path of the dataset")
-    parser.add_argument("--version", "-v", default=metadrive.constants.DATA_VERSION,  help="version")
-    parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it")
-    args = parser.parse_args()
-
-    force_overwrite = True
-    dataset_name = args.dataset_name
-    output_path = args.dataset_path
-    version = args.version
-
-    scenario_indices, env = get_pg_scenarios(100, IDMPolicy)
-
-    write_to_directory(
-        convert_func=convert_pg_scenario,
-        scenarios=scenario_indices,
-        output_path=output_path,
-        dataset_version=version,
-        dataset_name=dataset_name,
-        force_overwrite=force_overwrite,
-        env=env
-    )

scenarionet/tests/local_test/_test_convert_waymo.py

@@ -1,35 +0,0 @@
-import argparse
-import logging
-import os
-
-from scenarionet import SCENARIONET_DATASET_PATH
-from scenarionet.converter.utils import write_to_directory
-from scenarionet.converter.waymo.utils import convert_waymo_scenario, get_waymo_scenarios
-
-logger = logging.getLogger(__name__)
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--dataset_name", "-n", default="waymo",
-                        help="Dataset name, will be used to generate scenario files")
-    parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "waymo"),
-                        help="The path of the dataset")
-    parser.add_argument("--version", "-v", default='v1.2',  help="version")
-    args = parser.parse_args()
-
-    force_overwrite = True
-    dataset_name = args.dataset_name
-    output_path = args.dataset_path
-    version = args.version
-
-    waymo_data_directory = os.path.join(SCENARIONET_DATASET_PATH, "../waymo_origin")
-    scenarios = get_waymo_scenarios(waymo_data_directory, num_workers=3)
-
-    write_to_directory(
-        convert_func=convert_waymo_scenario,
-        scenarios=scenarios,
-        output_path=output_path,
-        dataset_version=version,
-        dataset_name=dataset_name,
-        force_overwrite=force_overwrite,
-        num_workers=8)

scenarionet/tests/local_test/generate_data.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-nohup python _test_convert_nuplan.py > nuplan.log 2>&1 &
+nohup python ../../scripts/convert_nuplan.py --overwrite > nuplan.log 2>&1 &
-nohup python _test_convert_nuscenes.py > nuscenes.log 2>&1 &
+nohup python ../../scripts/convert_nuscenes.py --overwrite > nuscenes.log 2>&1 &
-nohup python _test_convert_pg.py > pg.log 2>&1 &
+nohup python ../../scripts/convert_pg.py --overwrite  > pg.log 2>&1 &
-nohup python _test_convert_waymo.py > waymo.log 2>&1 &
+nohup python ../../scripts/convert_waymo.py --overwrite > waymo.log 2>&1 &