diff --git a/scenarionet/builder/utils.py b/scenarionet/builder/utils.py index ca51870..35bc678 100644 --- a/scenarionet/builder/utils.py +++ b/scenarionet/builder/utils.py @@ -81,7 +81,11 @@ def combine_multiple_dataset( else: with open(osp.join(abs_dir_path, ScenarioDescription.DATASET.MAPPING_FILE), "rb+") as f: mapping = pickle.load(f) - new_mapping = {k: os.path.relpath(abs_dir_path, output_abs_path) for k, v in mapping.items()} + new_mapping = {} + for file, rel_path in mapping.items(): + # mapping to real file path + new_mapping[file] = os.path.relpath(osp.join(abs_dir_path, rel_path), output_abs_path) + mappings.update(new_mapping) # apply filter stage @@ -98,5 +102,3 @@ def combine_multiple_dataset( save_summary_anda_mapping(summary_file, mapping_file, summaries, mappings) return summaries, mappings - - diff --git a/scenarionet/scripts/verify_scenarios.py b/scenarionet/scripts/verify_scenarios.py new file mode 100644 index 0000000..e69de29 diff --git a/scenarionet/tests/local_test/_test_convert_nuplan.py b/scenarionet/tests/local_test/_test_convert_nuplan.py deleted file mode 100644 index c77f05d..0000000 --- a/scenarionet/tests/local_test/_test_convert_nuplan.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -This script aims to convert nuplan scenarios to ScenarioDescription, so that we can load any nuplan scenarios into -MetaDrive. -""" -import argparse -import os - -from scenarionet import SCENARIONET_DATASET_PATH -from scenarionet.converter.nuplan.utils import get_nuplan_scenarios, convert_nuplan_scenario -from scenarionet.converter.utils import write_to_directory - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--dataset_name", "-n", default="nuplan", - help="Dataset name, will be used to generate scenario files") - parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "nuplan"), - help="The path of the dataset") - parser.add_argument("--version", "-v", default='v1.1', help="version") - parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it") - args = parser.parse_args() - - force_overwrite = True - dataset_name = args.dataset_name - output_path = args.dataset_path - version = args.version - - data_root = os.path.join(os.getenv("NUPLAN_DATA_ROOT"), "nuplan-v1.1/splits/mini") - map_root = os.getenv("NUPLAN_MAPS_ROOT") - scenarios = get_nuplan_scenarios(data_root, map_root, logs=["2021.07.16.20.45.29_veh-35_01095_01486"]) - - write_to_directory( - convert_func=convert_nuplan_scenario, - scenarios=scenarios, - output_path=output_path, - dataset_version=version, - dataset_name=dataset_name, - force_overwrite=force_overwrite, - num_workers=8, - ) diff --git a/scenarionet/tests/local_test/_test_convert_nuscenes.py b/scenarionet/tests/local_test/_test_convert_nuscenes.py deleted file mode 100644 index 662ee59..0000000 --- a/scenarionet/tests/local_test/_test_convert_nuscenes.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -This script aims to convert nuscenes scenarios to ScenarioDescription, so that we can load any nuscenes scenarios into -MetaDrive. -""" -import argparse -import os.path - -from scenarionet import SCENARIONET_DATASET_PATH -from scenarionet.converter.nuscenes.utils import convert_nuscenes_scenario, get_nuscenes_scenarios -from scenarionet.converter.utils import write_to_directory - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--dataset_name", "-n", default="nuscenes", - help="Dataset name, will be used to generate scenario files") - parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "nuscenes"), - help="The path of the dataset") - parser.add_argument("--version", "-v", default='v1.0-mini', help="version") - parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it") - args = parser.parse_args() - - force_overwrite = True - dataset_name = args.dataset_name - output_path = args.dataset_path - version = args.version - - dataroot = '/home/shady/data/nuscenes' - scenarios, nusc = get_nuscenes_scenarios(dataroot, version) - - write_to_directory( - convert_func=convert_nuscenes_scenario, - scenarios=scenarios, - output_path=output_path, - dataset_version=version, - dataset_name=dataset_name, - force_overwrite=force_overwrite, - nuscenes=nusc, - num_workers=4 - ) diff --git a/scenarionet/tests/local_test/_test_convert_pg.py b/scenarionet/tests/local_test/_test_convert_pg.py deleted file mode 100644 index c4b31e7..0000000 --- a/scenarionet/tests/local_test/_test_convert_pg.py +++ /dev/null @@ -1,37 +0,0 @@ -# from metadrive.policy.expert_policy import ExpertPolicy -import argparse -import os.path - -import metadrive -from metadrive.policy.idm_policy import IDMPolicy - -from scenarionet import SCENARIONET_DATASET_PATH -from scenarionet.converter.pg.utils import get_pg_scenarios, convert_pg_scenario -from scenarionet.converter.utils import write_to_directory - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--dataset_name", "-n", default="pg", - help="Dataset name, will be used to generate scenario files") - parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "pg"), - help="The path of the dataset") - parser.add_argument("--version", "-v", default=metadrive.constants.DATA_VERSION, help="version") - parser.add_argument("--overwrite", action="store_true", help="If the dataset_path exists, overwrite it") - args = parser.parse_args() - - force_overwrite = True - dataset_name = args.dataset_name - output_path = args.dataset_path - version = args.version - - scenario_indices, env = get_pg_scenarios(100, IDMPolicy) - - write_to_directory( - convert_func=convert_pg_scenario, - scenarios=scenario_indices, - output_path=output_path, - dataset_version=version, - dataset_name=dataset_name, - force_overwrite=force_overwrite, - env=env - ) diff --git a/scenarionet/tests/local_test/_test_convert_waymo.py b/scenarionet/tests/local_test/_test_convert_waymo.py deleted file mode 100644 index cc80fca..0000000 --- a/scenarionet/tests/local_test/_test_convert_waymo.py +++ /dev/null @@ -1,35 +0,0 @@ -import argparse -import logging -import os - -from scenarionet import SCENARIONET_DATASET_PATH -from scenarionet.converter.utils import write_to_directory -from scenarionet.converter.waymo.utils import convert_waymo_scenario, get_waymo_scenarios - -logger = logging.getLogger(__name__) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--dataset_name", "-n", default="waymo", - help="Dataset name, will be used to generate scenario files") - parser.add_argument("--dataset_path", "-d", default=os.path.join(SCENARIONET_DATASET_PATH, "waymo"), - help="The path of the dataset") - parser.add_argument("--version", "-v", default='v1.2', help="version") - args = parser.parse_args() - - force_overwrite = True - dataset_name = args.dataset_name - output_path = args.dataset_path - version = args.version - - waymo_data_directory = os.path.join(SCENARIONET_DATASET_PATH, "../waymo_origin") - scenarios = get_waymo_scenarios(waymo_data_directory, num_workers=3) - - write_to_directory( - convert_func=convert_waymo_scenario, - scenarios=scenarios, - output_path=output_path, - dataset_version=version, - dataset_name=dataset_name, - force_overwrite=force_overwrite, - num_workers=8) diff --git a/scenarionet/tests/local_test/generate_data.sh b/scenarionet/tests/local_test/generate_data.sh index 7ed8e18..03dee06 100644 --- a/scenarionet/tests/local_test/generate_data.sh +++ b/scenarionet/tests/local_test/generate_data.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -nohup python _test_convert_nuplan.py > nuplan.log 2>&1 & -nohup python _test_convert_nuscenes.py > nuscenes.log 2>&1 & -nohup python _test_convert_pg.py > pg.log 2>&1 & -nohup python _test_convert_waymo.py > waymo.log 2>&1 & \ No newline at end of file +nohup python ../../scripts/convert_nuplan.py --overwrite > nuplan.log 2>&1 & +nohup python ../../scripts/convert_nuscenes.py --overwrite > nuscenes.log 2>&1 & +nohup python ../../scripts/convert_pg.py --overwrite > pg.log 2>&1 & +nohup python ../../scripts/convert_waymo.py --overwrite > waymo.log 2>&1 & \ No newline at end of file