Add come updates for Neurips paper (#4)

* scenarionet training * wandb * train utils * fix callback * run PPO * use pg test * save path * use torch * add dependency * update ignore * update training * large model * use curriculum training * add time to exp name * storage_path * restore * update training * use my key * add log message * check seed * restore callback * restore call bacl * add log message * add logging message * restore ray1.4 * length 500 * ray 100 * wandb * use tf * more levels * add callback * 10 worker * show level * no env horizon * callback result level * more call back * add diffuculty * add mroen stat * mroe stat * show levels * add callback * new * ep len 600 * fix setup * fix stepup * fix to 3.8 * update setup * parallel worker! * new exp * add callback * lateral dist * pg dataset * evaluate * modify config * align config * train single RL * update training script * 100w eval * less eval to reveal * 2000 env eval * new trianing * eval 1000 * update eval * more workers * more worker * 20 worker * dataset to database * split tool! * split dataset * try fix * train 003 * fix mapping * fix test * add waymo tqdm * utils * fix bug * fix bug * waymo * int type * 8 worker read * disable * read file * add log message * check existence * dist 0 * int * check num * suprass warning * add filter API * filter * store map false * new * ablation * filter * fix * update filyter * reanme to from * random select * add overlapping checj * fix * new training sceheme * new reward * add waymo train script * waymo different config * copy raw data * fix bug * add tqdm * update readme * waymo * pg * max lateral dist 3 * pg * crash_done instead of penalty * no crash done * gpu * update eval script * steering range penalty * evaluate * finish pg * update setup * fix bug * test * fix * add on line * train nuplan * generate sensor * udpate training * static obj * multi worker eval * filx bug * use ray for testing * eval! * filter senario * id filter * fox bug * dist = 2 * filter * eval * eval ret * ok * update training pg * test before use * store data=False * collect figures * capture pic --------- Co-authored-by: Quanyi Li <quanyi@bolei-gpu02.cs.ucla.edu>
2023-06-10 18:56:33 +01:00
parent 41c0b01f39
commit db50bca7fd
53 changed files with 2274 additions and 133 deletions
--- a/scenarionet_training/scripts/multi_worker_eval.py
+++ b/scenarionet_training/scripts/multi_worker_eval.py
@@ -0,0 +1,74 @@
+import argparse
+import pickle
+import json
+import os
+
+import numpy as np
+
+from scenarionet_training.scripts.train_nuplan import config
+from scenarionet_training.train_utils.callbacks import DrivingCallbacks
+from scenarionet_training.train_utils.multi_worker_PPO import MultiWorkerPPO
+from scenarionet_training.train_utils.utils import initialize_ray
+
+
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        elif isinstance(obj, np.int32):
+            return int(obj)
+        elif isinstance(obj, np.int64):
+            return int(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--start_index", type=int, default=0)
+    parser.add_argument("--ckpt_path", type=str, required=True)
+    parser.add_argument("--database_path", type=str, required=True)
+    parser.add_argument("--id", type=str, default="")
+    parser.add_argument("--num_scenarios", type=int, default=5000)
+    parser.add_argument("--num_workers", type=int, default=10)
+    parser.add_argument("--horizon", type=int, default=600)
+    parser.add_argument("--allowed_more_steps", type=int, default=50)
+    parser.add_argument("--max_lateral_dist", type=int, default=2.5)
+    parser.add_argument("--overwrite", action="store_true")
+
+    args = parser.parse_args()
+    file = "eval_{}_{}_{}".format(args.id, os.path.basename(args.ckpt_path), os.path.basename(args.database_path))
+    if os.path.exists(file) and not args.overwrite:
+        raise FileExistsError("Please remove {} or set --overwrite".format(file))
+    initialize_ray(test_mode=True, num_gpus=1)
+
+    config["callbacks"] = DrivingCallbacks
+    config["evaluation_num_workers"] = args.num_workers
+    config["evaluation_num_episodes"] = args.num_scenarios
+    config["metrics_smoothing_episodes"] = args.num_scenarios
+    config["custom_eval_function"] = None
+    config["num_workers"] = 0
+    config["evaluation_config"]["env_config"].update(dict(
+        start_scenario_index=args.start_index,
+        num_scenarios=args.num_scenarios,
+        sequential_seed=True,
+        store_map=False,
+        store_data=False,
+        allowed_more_steps=args.allowed_more_steps,
+        # no_map=True,
+        max_lateral_dist=args.max_lateral_dist,
+        curriculum_level=1,  # disable curriculum
+        target_success_rate=1,
+        horizon=args.horizon,
+        episodes_to_evaluate_curriculum=args.num_scenarios,
+        data_directory=args.database_path,
+        use_render=False))
+
+    trainer = MultiWorkerPPO(config)
+    trainer.restore(args.ckpt_path)
+
+    ret = trainer._evaluate()["evaluation"]
+    with open(file + ".json", "w") as f:
+        json.dump(ret, f, cls=NumpyEncoder)
+
+    with open(file + ".pkl", "wb+") as f:
+        pickle.dump(ret, f)