train.py 11.5 KB
Newer Older
metataro's avatar
metataro committed
1 2 3
#!/usr/bin/env python

import os
4
import numpy as np
metataro's avatar
metataro committed
5 6

import ray
MasterScrat's avatar
MasterScrat committed
7
import yaml
8
from pathlib import Path
metataro's avatar
metataro committed
9 10 11
from ray.cluster_utils import Cluster
from ray.rllib.evaluation import MultiAgentEpisode
from ray.rllib.utils.framework import try_import_tf, try_import_torch
12
from ray.tune import run_experiments, Experiment
13
from ray.tune.logger import TBXLogger
MasterScrat's avatar
MasterScrat committed
14 15
from ray.tune.resources import resources_to_json
from ray.tune.tune import _make_scheduler
metataro's avatar
metataro committed
16

17 18 19 20 21
from ray.rllib.agents.trainer import with_common_config
from ray.rllib.agents.ppo.ppo import PPOTrainer

from algorithms.imitation_agent.imitation_trainer import ImitationAgent

MasterScrat's avatar
MasterScrat committed
22
from utils.argparser import create_parser
23
from utils.loader import load_envs, load_models, load_algorithms
nilabha's avatar
nilabha committed
24 25 26 27

from envs.flatland import get_eval_config
from ray.rllib.utils import merge_dicts

MasterScrat's avatar
MasterScrat committed
28 29
# Custom wandb logger with hotfix to allow custom callbacks
from wandblogger import WandbLogger
metataro's avatar
metataro committed
30 31 32 33 34 35 36 37

# Try to import both backends for flag checking/warnings.
tf = try_import_tf()
torch, _ = try_import_torch()

# Register all necessary assets in tune registries
load_envs(os.getcwd())  # Load envs
load_models(os.getcwd())  # Load models
38 39
from algorithms import CUSTOM_ALGORITHMS
load_algorithms(CUSTOM_ALGORITHMS)  # Load algorithms
metataro's avatar
metataro committed
40

41
MAX_ITERATIONS = 1000000
metataro's avatar
metataro committed
42 43 44 45 46 47 48 49 50

def on_episode_end(info):
    episode = info["episode"]  # type: MultiAgentEpisode

    episode_steps = 0
    episode_max_steps = 0
    episode_num_agents = 0
    episode_score = 0
    episode_done_agents = 0
51
    episode_num_swaps = 0
metataro's avatar
metataro committed
52 53 54 55 56 57 58

    for agent, agent_info in episode._agent_to_last_info.items():
        if episode_max_steps == 0:
            episode_max_steps = agent_info["max_episode_steps"]
            episode_num_agents = agent_info["num_agents"]
        episode_steps = max(episode_steps, agent_info["agent_step"])
        episode_score += agent_info["agent_score"]
59 60
        if "num_swaps" in agent_info:
            episode_num_swaps += agent_info["num_swaps"]
metataro's avatar
metataro committed
61 62 63
        if agent_info["agent_done"]:
            episode_done_agents += 1

64 65
    # Not a valid check when considering a single policy for multiple agents
    #assert len(episode._agent_to_last_info) == episode_num_agents
metataro's avatar
metataro committed
66

nilabha's avatar
nilabha committed
67
    norm_factor = 1.0 / (episode_max_steps * episode_num_agents)
metataro's avatar
metataro committed
68 69 70 71 72 73 74 75
    percentage_complete = float(episode_done_agents) / episode_num_agents

    episode.custom_metrics["episode_steps"] = episode_steps
    episode.custom_metrics["episode_max_steps"] = episode_max_steps
    episode.custom_metrics["episode_num_agents"] = episode_num_agents
    episode.custom_metrics["episode_return"] = episode.total_reward
    episode.custom_metrics["episode_score"] = episode_score
    episode.custom_metrics["episode_score_normalized"] = episode_score * norm_factor
76
    episode.custom_metrics["episode_num_swaps"] = episode_num_swaps / 2
metataro's avatar
metataro committed
77 78 79
    episode.custom_metrics["percentage_complete"] = percentage_complete


80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
def imitation_ppo_train_fn(config,reporter=None):
    imitation_trainer = ImitationAgent(config,
    env=config.get("env"),)

    ppo_trainer = PPOTrainer(config,
    env=config.get("env"),)

    expert_ratio = config.get("env_config",{}).get("expert",{}).get('ratio', 0.5)
    expert_min_ratio = config.get("env_config",{}).get("expert",{}).get('min_ratio', expert_ratio)
    expert_ratio_decay = config.get("env_config",{}).get("expert",{}).get('ratio_decay', 1)

    for i in range(MAX_ITERATIONS):

        print("== Iteration", i, "==")

        trainer_type = np.random.binomial(size=1, n=1, p= expert_ratio)[0]

        if trainer_type:
            # improve the Imitation policy
            print("-- Imitation --")
            result_imitate = imitation_trainer.train()
            if reporter:
                reporter(**result_imitate)
            if i % checkpoint_freq == 0:
                checkpoint = imitation_trainer.save()
                print("checkpoint saved at", checkpoint)

            ppo_trainer.set_weights(imitation_trainer.get_weights())

        else:
            # improve the PPO policy
            print("-- PPO --")
            result_ppo = ppo_trainer.train()
            if reporter:
                reporter(**result_ppo)
            if i % checkpoint_freq == 0:
                checkpoint = ppo_trainer.save()
                print("checkpoint saved at", checkpoint)

        expert_ratio = max(expert_min_ratio, expert_ratio_decay * expert_ratio)

    imitation_trainer.stop()
    ppo_trainer.stop()

    print("Completed: OK")


metataro's avatar
metataro committed
127 128 129 130 131 132 133 134 135 136 137 138 139 140
def run(args, parser):
    if args.config_file:
        with open(args.config_file) as f:
            experiments = yaml.safe_load(f)
    else:
        # Note: keep this in sync with tune/config_parser.py
        experiments = {
            args.experiment_name: {  # i.e. log to ~/ray_results/default
                "run": args.run,
                "checkpoint_freq": args.checkpoint_freq,
                "keep_checkpoints_num": args.keep_checkpoints_num,
                "checkpoint_score_attr": args.checkpoint_score_attr,
                "local_dir": args.local_dir,
                "resources_per_trial": (
MasterScrat's avatar
MasterScrat committed
141 142
                        args.resources_per_trial and
                        resources_to_json(args.resources_per_trial)),
metataro's avatar
metataro committed
143 144 145 146 147 148 149 150 151
                "stop": args.stop,
                "config": dict(args.config, env=args.env),
                "restore": args.restore,
                "num_samples": args.num_samples,
                "upload_dir": args.upload_dir,
            }
        }

    verbose = 1
152
    custom_fn = False
MasterScrat's avatar
MasterScrat committed
153
    webui_host = "localhost"
metataro's avatar
metataro committed
154 155 156 157
    for exp in experiments.values():
        # Bazel makes it hard to find files specified in `args` (and `data`).
        # Look for them here.
        # NOTE: Some of our yaml files don't have a `config` section.
158 159 160 161 162 163 164
        if exp.get("config", {}).get("input"):
            if not isinstance(exp.get("config", {}).get("input"),dict):
                if not os.path.exists(exp["config"]["input"]):
                    # This script runs in the ray/rllib dir.
                    rllib_dir = Path(__file__).parent
                    input_file = rllib_dir.absolute().joinpath(exp["config"]["input"])
                    exp["config"]["input"] = str(input_file)
metataro's avatar
metataro committed
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183

        if not exp.get("run"):
            parser.error("the following arguments are required: --run")
        if not exp.get("env") and not exp.get("config", {}).get("env"):
            parser.error("the following arguments are required: --env")
        if args.eager:
            exp["config"]["eager"] = True
        if args.torch:
            exp["config"]["use_pytorch"] = True
        if args.v:
            exp["config"]["log_level"] = "INFO"
            verbose = 2
        if args.vv:
            exp["config"]["log_level"] = "DEBUG"
            verbose = 3
        if args.trace:
            if not exp["config"].get("eager"):
                raise ValueError("Must enable --eager to enable tracing.")
            exp["config"]["eager_tracing"] = True
MasterScrat's avatar
MasterScrat committed
184 185
        if args.bind_all:
            webui_host = "0.0.0.0"
metataro's avatar
metataro committed
186 187 188 189
        if args.log_flatland_stats:
            exp['config']['callbacks'] = {
                'on_episode_end': on_episode_end,
            }
nilabha's avatar
nilabha committed
190 191

        if args.eval:
192 193 194 195 196 197
            eval_configs_file = exp['config'].get('env_config',\
                                    {}).get('eval_generator',"default")
            if args.record:
                eval_configs_file = exp['config'].get('env_config',\
                        {}).get('eval_generator',"default_render")
            eval_configs = get_eval_config(eval_configs_file)
nilabha's avatar
nilabha committed
198
            eval_seed = eval_configs.get('evaluation_config',{}).get('env_config',{}).get('seed')
199
            eval_render = eval_configs.get('evaluation_config',{}).get('env_config',{}).get('render')
nilabha's avatar
nilabha committed
200 201 202 203 204 205 206 207 208

            # add evaluation config to the current config
            exp['config'] = merge_dicts(exp['config'],eval_configs)
            if exp['config'].get('evaluation_config'):
                exp['config']['evaluation_config']['env_config'] = exp['config'].get('env_config')
                eval_env_config = exp['config']['evaluation_config'].get('env_config')
                if eval_seed and eval_env_config:
                    # We override the env seed from the evaluation config
                    eval_env_config['seed'] = eval_seed
209 210 211
                if eval_render and eval_env_config:
                    # We override the env render from the evaluation config
                    eval_env_config['render'] = eval_render
212 213 214 215
                    # Set video_dir if it exists
                    eval_render_dir = eval_configs.get('evaluation_config',{}).get('env_config',{}).get('video_dir')
                    if eval_render_dir:
                        eval_env_config['video_dir'] = eval_render_dir
nilabha's avatar
nilabha committed
216 217 218 219 220 221 222 223
                # Remove any wandb related configs
                if eval_env_config:
                    if eval_env_config.get('wandb'):
                        del eval_env_config['wandb']

            # Remove any wandb related configs
            if exp['config']['evaluation_config'].get('wandb'):
                del exp['config']['evaluation_config']['wandb']
224 225
        if args.custom_fn:
            custom_fn = globals()[exp['config'].get("env_config",{}).get("custom_fn","imitation_ppo_train_fn")]
226 227
        if args.save_checkpoint:
            exp['config']['env_config']['save_checkpoint'] = True
MasterScrat's avatar
MasterScrat committed
228 229 230
        if args.config_file:
            # TODO should be in exp['config'] directly
            exp['config']['env_config']['yaml_config'] = args.config_file
231
        exp['loggers'] = [WandbLogger, TBXLogger]
metataro's avatar
metataro committed
232

233 234 235 236 237 238 239 240
        global checkpoint_freq,keep_checkpoints_num,checkpoint_score_attr,checkpoint_at_end
        checkpoint_freq = exp['checkpoint_freq']

        # TODO: Below checkpoints paramaters are not supported for default custom_fn
        keep_checkpoints_num = exp['keep_checkpoints_num']
        checkpoint_score_attr = exp['checkpoint_score_attr']
        checkpoint_at_end = exp['checkpoint_at_end']

metataro's avatar
metataro committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
    if args.ray_num_nodes:
        cluster = Cluster()
        for _ in range(args.ray_num_nodes):
            cluster.add_node(
                num_cpus=args.ray_num_cpus or 1,
                num_gpus=args.ray_num_gpus or 0,
                object_store_memory=args.ray_object_store_memory,
                memory=args.ray_memory,
                redis_max_memory=args.ray_redis_max_memory)
        ray.init(address=cluster.address)
    else:
        ray.init(
            address=args.ray_address,
            object_store_memory=args.ray_object_store_memory,
            memory=args.ray_memory,
            redis_max_memory=args.ray_redis_max_memory,
            num_cpus=args.ray_num_cpus,
MasterScrat's avatar
MasterScrat committed
258 259 260
            num_gpus=args.ray_num_gpus,
            webui_host=webui_host)

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
    if custom_fn:
        for exp in experiments.values():
            configs = with_common_config(exp["config"])
            configs['env'] = exp.get('env')
            resources = PPOTrainer.default_resource_request(configs).to_json()
            experiment_spec = Experiment(
                                custom_fn.__name__,
                                custom_fn,
                                resources_per_trial=resources,
                                config=configs,
                                stop=exp.get('stop'),
                                num_samples=exp.get('num_samples',1),
                                loggers=exp.get('loggers'),
                                restore=None)
        experiments = experiment_spec

metataro's avatar
metataro committed
277 278 279 280 281 282 283 284 285 286 287 288 289
    run_experiments(
        experiments,
        scheduler=_make_scheduler(args),
        queue_trials=args.queue_trials,
        resume=args.resume,
        verbose=verbose,
        concurrent=True)


if __name__ == "__main__":
    parser = create_parser()
    args = parser.parse_args()
    run(args, parser)