Add files via upload
This commit is contained in:
Nguyen Viet Tuan
@@ -1,383 +1,300 @@
|
||||
import itertools
|
||||
import logging
|
||||
import numpy as np
|
||||
import tensorflow
|
||||
import tensorflow.compat.v1 as tf
|
||||
import time
|
||||
import os
|
||||
import pandas as pd
|
||||
import subprocess
|
||||
import random
|
||||
import tensorflow as tf
|
||||
|
||||
"""
|
||||
initializers
|
||||
"""
|
||||
DEFAULT_SCALE = np.sqrt(2)
|
||||
DEFAULT_MODE = 'fan_in'
|
||||
|
||||
|
||||
def check_dir(cur_dir):
|
||||
if not os.path.exists(cur_dir):
|
||||
return False
|
||||
return True
|
||||
def ortho_init(scale=DEFAULT_SCALE, mode=None):
|
||||
def _ortho_init(shape, dtype, partition_info=None):
|
||||
# lasagne ortho init for tf
|
||||
shape = tuple(shape)
|
||||
if len(shape) == 2: # fc: in, out
|
||||
flat_shape = shape
|
||||
elif (len(shape) == 3) or (len(shape) == 4): # 1d/2dcnn: (in_h), in_w, in_c, out
|
||||
flat_shape = (np.prod(shape[:-1]), shape[-1])
|
||||
a = np.random.standard_normal(flat_shape)
|
||||
u, _, v = np.linalg.svd(a, full_matrices=False)
|
||||
q = u if u.shape == flat_shape else v # pick the one with the correct shape
|
||||
q = q.reshape(shape)
|
||||
return (scale * q).astype(np.float32)
|
||||
return _ortho_init
|
||||
|
||||
|
||||
def copy_file(src_dir, tar_dir):
|
||||
cmd = 'cp %s %s' % (src_dir, tar_dir)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
def norm_init(scale=DEFAULT_SCALE, mode=DEFAULT_MODE):
|
||||
def _norm_init(shape, dtype, partition_info=None):
|
||||
shape = tuple(shape)
|
||||
if len(shape) == 2:
|
||||
n_in = shape[0]
|
||||
elif (len(shape) == 3) or (len(shape) == 4):
|
||||
n_in = np.prod(shape[:-1])
|
||||
a = np.random.standard_normal(shape)
|
||||
if mode == 'fan_in':
|
||||
n = n_in
|
||||
elif mode == 'fan_out':
|
||||
n = shape[-1]
|
||||
elif mode == 'fan_avg':
|
||||
n = 0.5 * (n_in + shape[-1])
|
||||
return (scale * a / np.sqrt(n)).astype(np.float32)
|
||||
|
||||
|
||||
def find_file(cur_dir, suffix='.ini'):
|
||||
for file in os.listdir(cur_dir):
|
||||
if file.endswith(suffix):
|
||||
return cur_dir + '/' + file
|
||||
logging.error('Cannot find %s file' % suffix)
|
||||
return None
|
||||
DEFAULT_METHOD = ortho_init
|
||||
"""
|
||||
layers
|
||||
"""
|
||||
|
||||
|
||||
def init_dir(base_dir, pathes=['log', 'data', 'model']):
|
||||
if not os.path.exists(base_dir):
|
||||
os.mkdir(base_dir)
|
||||
dirs = {}
|
||||
for path in pathes:
|
||||
cur_dir = base_dir + '/%s/' % path
|
||||
if not os.path.exists(cur_dir):
|
||||
os.mkdir(cur_dir)
|
||||
dirs[path] = cur_dir
|
||||
return dirs
|
||||
def conv(x, scope, n_out, f_size, stride=1, pad='VALID', f_size_w=None, act=tf.nn.relu,
|
||||
conv_dim=1, init_scale=DEFAULT_SCALE, init_mode=None, init_method=DEFAULT_METHOD):
|
||||
with tf.variable_scope(scope):
|
||||
b = tf.get_variable(
|
||||
"b", [n_out], initializer=tf.constant_initializer(0.0))
|
||||
if conv_dim == 1:
|
||||
n_c = x.shape[2].value
|
||||
w = tf.get_variable("w", [f_size, n_c, n_out],
|
||||
initializer=init_method(init_scale, init_mode))
|
||||
z = tf.nn.conv1d(x, w, stride=stride, padding=pad) + b
|
||||
elif conv_dim == 2:
|
||||
n_c = x.shape[3].value
|
||||
if f_size_w is None:
|
||||
f_size_w = f_size
|
||||
w = tf.get_variable("w", [f_size, f_size_w, n_c, n_out],
|
||||
initializer=init_method(init_scale, init_mode))
|
||||
z = tf.nn.conv2d(
|
||||
x, w, strides=[1, stride, stride, 1], padding=pad) + b
|
||||
return act(z)
|
||||
|
||||
|
||||
def init_log(log_dir):
|
||||
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
|
||||
level=logging.INFO,
|
||||
handlers=[
|
||||
logging.FileHandler('%s/%d.log' %
|
||||
(log_dir, time.time())),
|
||||
logging.StreamHandler()
|
||||
])
|
||||
def fc(x, scope, n_out, act=tf.nn.relu, init_scale=DEFAULT_SCALE,
|
||||
init_mode=DEFAULT_MODE, init_method=DEFAULT_METHOD):
|
||||
with tf.compat.v1.variable_scope(scope):
|
||||
n_in = x.shape[1].value
|
||||
w = tf.compat.v1.get_variable("w", [n_in, n_out],
|
||||
initializer=init_method(init_scale, init_mode))
|
||||
b = tf.compat.v1.get_variable(
|
||||
"b", [n_out], initializer=tf.constant_initializer(0.0))
|
||||
z = tf.matmul(x, w) + b
|
||||
return act(z)
|
||||
|
||||
|
||||
def init_test_flag(test_mode):
|
||||
if test_mode == 'no_test':
|
||||
return False, False
|
||||
if test_mode == 'in_train_test':
|
||||
return True, False
|
||||
if test_mode == 'after_train_test':
|
||||
return False, True
|
||||
if test_mode == 'all_test':
|
||||
return True, True
|
||||
return False, False
|
||||
def batch_to_seq(x):
|
||||
n_step = x.shape[0].value
|
||||
if len(x.shape) == 1:
|
||||
x = tf.expand_dims(x, -1)
|
||||
return tf.split(axis=0, num_or_size_splits=n_step, value=x)
|
||||
|
||||
|
||||
class Counter:
|
||||
def __init__(self, total_step, test_step, log_step):
|
||||
self.counter = itertools.count(1)
|
||||
self.cur_step = 0
|
||||
self.cur_test_step = 0
|
||||
self.total_step = total_step
|
||||
self.test_step = test_step
|
||||
self.log_step = log_step
|
||||
self.stop = False
|
||||
# self.init_test = True
|
||||
|
||||
def next(self):
|
||||
self.cur_step = next(self.counter)
|
||||
return self.cur_step
|
||||
|
||||
def should_test(self):
|
||||
test = False
|
||||
if (self.cur_step - self.cur_test_step) >= self.test_step:
|
||||
test = True
|
||||
self.cur_test_step = self.cur_step
|
||||
return test
|
||||
|
||||
def should_log(self):
|
||||
return (self.cur_step % self.log_step == 0)
|
||||
|
||||
def should_stop(self):
|
||||
if self.cur_step >= self.total_step:
|
||||
return True
|
||||
return self.stop
|
||||
def seq_to_batch(x):
|
||||
return tf.concat(axis=0, values=x)
|
||||
|
||||
|
||||
class Trainer():
|
||||
def __init__(self, env, model, global_counter, summary_writer, run_test, output_path=None):
|
||||
self.cur_step = 0
|
||||
self.global_counter = global_counter
|
||||
self.env = env
|
||||
self.agent = self.env.agent
|
||||
self.model = model
|
||||
self.sess = self.model.sess
|
||||
self.n_step = self.model.n_step
|
||||
self.summary_writer = summary_writer
|
||||
self.run_test = run_test
|
||||
self.data = []
|
||||
self.output_path = output_path
|
||||
if run_test:
|
||||
self.test_num = self.env.test_num
|
||||
logging.info('Testing: total test num: %d' % self.test_num)
|
||||
self._init_summary()
|
||||
def lstm(xs, dones, s, scope, init_scale=DEFAULT_SCALE, init_mode=DEFAULT_MODE,
|
||||
init_method=DEFAULT_METHOD):
|
||||
xs = batch_to_seq(xs)
|
||||
# need dones to reset states
|
||||
dones = batch_to_seq(dones)
|
||||
n_in = xs[0].shape[1].value
|
||||
n_out = s.shape[0] // 2
|
||||
with tf.compat.v1.variable_scope(scope):
|
||||
wx = tf.compat.v1.get_variable("wx", [n_in, n_out*4],
|
||||
initializer=init_method(init_scale, init_mode))
|
||||
wh = tf.compat.v1.get_variable("wh", [n_out, n_out*4],
|
||||
initializer=init_method(init_scale, init_mode))
|
||||
b = tf.compat.v1.get_variable(
|
||||
"b", [n_out*4], initializer=tf.constant_initializer(0.0))
|
||||
s = tf.expand_dims(s, 0)
|
||||
c, h = tf.split(axis=1, num_or_size_splits=2, value=s)
|
||||
for ind, (x, done) in enumerate(zip(xs, dones)):
|
||||
c = c * (1-done)
|
||||
h = h * (1-done)
|
||||
z = tf.matmul(x, wx) + tf.matmul(h, wh) + b
|
||||
i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z)
|
||||
i = tf.nn.sigmoid(i)
|
||||
f = tf.nn.sigmoid(f)
|
||||
o = tf.nn.sigmoid(o)
|
||||
u = tf.tanh(u)
|
||||
c = f*c + i*u
|
||||
h = o*tf.tanh(c)
|
||||
xs[ind] = h
|
||||
s = tf.concat(axis=1, values=[c, h])
|
||||
return seq_to_batch(xs), tf.squeeze(s)
|
||||
|
||||
def _init_summary(self):
|
||||
self.train_reward = tf.placeholder(tensorflow.float32, [])
|
||||
self.train_summary = tf.summary.scalar(
|
||||
'train_reward', self.train_reward)
|
||||
self.test_reward = tf.placeholder(tensorflow.float32, [])
|
||||
self.test_summary = tf.summary.scalar('test_reward', self.test_reward)
|
||||
|
||||
def _add_summary(self, reward, global_step, is_train=True):
|
||||
if is_train:
|
||||
summ = self.sess.run(self.train_summary, {
|
||||
self.train_reward: reward})
|
||||
def test_layers():
|
||||
print(tf.__version__)
|
||||
tf.reset_default_graph()
|
||||
sess = tf.Session()
|
||||
n_step = 5
|
||||
fc_x = tf.placeholder(tf.float32, [None, 10])
|
||||
lstm_x = tf.placeholder(tf.float32, [n_step, 2])
|
||||
lstm_done = tf.placeholder(tf.float32, [n_step])
|
||||
lstm_s = tf.placeholder(tf.float32, [20])
|
||||
conv1_x = tf.placeholder(tf.float32, [None, 8, 1])
|
||||
conv2_x = tf.placeholder(tf.float32, [None, 8, 8, 1])
|
||||
fc_out = fc(fc_x, 'fc', 10)
|
||||
lstm_out, lstm_ns = lstm(lstm_x, lstm_done, lstm_s, 'lstm')
|
||||
conv1_out = conv(conv1_x, 'conv1', 10, 4, conv_dim=1)
|
||||
conv2_out = conv(conv2_x, 'conv2', 10, 4, conv_dim=2)
|
||||
sess.run(tf.global_variables_initializer())
|
||||
inputs = {'fc': {fc_x: np.random.randn(n_step, 10)},
|
||||
'lstm_done': {lstm_x: np.zeros((n_step, 2)),
|
||||
lstm_done: np.ones(n_step),
|
||||
lstm_s: np.random.randn(20)},
|
||||
'lstm': {lstm_x: np.random.randn(n_step, 2),
|
||||
lstm_done: np.zeros(n_step),
|
||||
lstm_s: np.random.randn(20)},
|
||||
'conv1': {conv1_x: np.random.randn(n_step, 8, 1)},
|
||||
'conv2': {conv2_x: np.random.randn(n_step, 8, 8, 1)}}
|
||||
outputs = {'fc': [fc_out], 'lstm_done': [lstm_out, lstm_ns],
|
||||
'conv1': [conv1_out], 'conv2': [conv2_out],
|
||||
'lstm': [lstm_out, lstm_ns]}
|
||||
for scope in ['fc', 'lstm', 'conv1', 'conv2']:
|
||||
print(scope)
|
||||
wts = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope)
|
||||
for wt in wts:
|
||||
wt_val = wt.eval(sess)
|
||||
print(wt_val.shape)
|
||||
print(np.mean(wt_val), np.std(wt_val),
|
||||
np.min(wt_val), np.max(wt_val))
|
||||
print('=====================================')
|
||||
for x_name in inputs:
|
||||
print(x_name)
|
||||
out = sess.run(outputs[x_name], inputs[x_name])
|
||||
if x_name.startswith('lstm'):
|
||||
print(out[0])
|
||||
print(out[1])
|
||||
else:
|
||||
summ = self.sess.run(self.test_summary, {self.test_reward: reward})
|
||||
self.summary_writer.add_summary(summ, global_step=global_step)
|
||||
print(out[0].shape)
|
||||
|
||||
def explore(self, prev_ob, prev_done):
|
||||
ob = prev_ob
|
||||
done = prev_done
|
||||
rewards = []
|
||||
for _ in range(self.n_step):
|
||||
policy, value = self.model.forward(ob, done)
|
||||
# need to update fingerprint before calling step
|
||||
self.env.update_fingerprint(policy)
|
||||
action = []
|
||||
for pi in policy:
|
||||
action.append(np.random.choice(
|
||||
np.arange(len(pi)), p=pi))
|
||||
|
||||
next_ob, reward, done, global_reward = self.env.step(action)
|
||||
rewards.append(global_reward)
|
||||
global_step = self.global_counter.next()
|
||||
self.cur_step += 1
|
||||
if self.agent.endswith('a2c'):
|
||||
self.model.add_transition(ob, action, reward, value, done)
|
||||
else:
|
||||
self.model.add_transition(ob, action, reward, next_ob, done)
|
||||
# logging
|
||||
if self.global_counter.should_log():
|
||||
logging.info('''Training: global step %d, episode step %d,
|
||||
ob: %s, a: %s, pi: %s, r: %.2f, train r: %.2f, done: %r''' %
|
||||
(global_step, self.cur_step,
|
||||
str(ob), str(action), str(policy), global_reward, np.mean(reward), done))
|
||||
"""
|
||||
buffers
|
||||
"""
|
||||
|
||||
if done:
|
||||
break
|
||||
ob = next_ob
|
||||
if self.agent.endswith('a2c'):
|
||||
if done:
|
||||
R = 0 if self.agent == 'a2c' else [0] * self.model.n_agent
|
||||
else:
|
||||
R = self.model.forward(ob, False, 'v')
|
||||
|
||||
class TransBuffer:
|
||||
def reset(self):
|
||||
self.buffer = []
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return len(self.buffer)
|
||||
|
||||
def add_transition(self, ob, a, r, *_args, **_kwargs):
|
||||
raise NotImplementedError()
|
||||
|
||||
def sample_transition(self, *_args, **_kwargs):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class OnPolicyBuffer(TransBuffer):
|
||||
def __init__(self, gamma):
|
||||
self.gamma = gamma
|
||||
self.reset()
|
||||
|
||||
def reset(self, done=False):
|
||||
# the done before each step is required
|
||||
self.obs = []
|
||||
self.acts = []
|
||||
self.rs = []
|
||||
self.vs = []
|
||||
self.dones = [done]
|
||||
|
||||
def add_transition(self, ob, a, r, v, done):
|
||||
self.obs.append(ob)
|
||||
self.acts.append(a)
|
||||
self.rs.append(r)
|
||||
self.vs.append(v)
|
||||
self.dones.append(done)
|
||||
|
||||
def _add_R_Adv(self, R):
|
||||
Rs = []
|
||||
Advs = []
|
||||
# use post-step dones here
|
||||
for r, v, done in zip(self.rs[::-1], self.vs[::-1], self.dones[:0:-1]):
|
||||
R = r + self.gamma * R * (1.-done)
|
||||
Adv = R - v
|
||||
Rs.append(R)
|
||||
Advs.append(Adv)
|
||||
Rs.reverse()
|
||||
Advs.reverse()
|
||||
self.Rs = Rs
|
||||
self.Advs = Advs
|
||||
|
||||
def sample_transition(self, R, discrete=True):
|
||||
self._add_R_Adv(R)
|
||||
obs = np.array(self.obs, dtype=np.float32)
|
||||
if discrete:
|
||||
acts = np.array(self.acts, dtype=np.int32)
|
||||
else:
|
||||
R = 0
|
||||
return ob, done, R, rewards
|
||||
|
||||
def inference(self, ob, policy_type='default'):
|
||||
# note this done is pre-decision to reset LSTM states!
|
||||
done = False
|
||||
# self.model.reset()
|
||||
# policy-based on-poicy learning
|
||||
policy = self.model.forward(ob, done, 'p')
|
||||
self.env.update_fingerprint(policy)
|
||||
action = []
|
||||
for pi in policy:
|
||||
if policy_type != 'deterministic':
|
||||
action.append(np.random.choice(
|
||||
np.arange(len(pi)), p=pi))
|
||||
else:
|
||||
action.append(np.argmax(np.array(pi)))
|
||||
|
||||
return action
|
||||
|
||||
def perform(self, policy_type='default'):
|
||||
ob = self.env.reset()
|
||||
# note this done is pre-decision to reset LSTM states!
|
||||
done = True
|
||||
self.model.reset()
|
||||
rewards = []
|
||||
while True:
|
||||
# policy-based on-poicy learning
|
||||
policy = self.model.forward(ob, done, 'p')
|
||||
self.env.update_fingerprint(policy)
|
||||
action = []
|
||||
for pi in policy:
|
||||
if policy_type != 'deterministic':
|
||||
action.append(np.random.choice(
|
||||
np.arange(len(pi)), p=pi))
|
||||
else:
|
||||
action.append(np.argmax(np.array(pi)))
|
||||
|
||||
next_ob, reward, done, global_reward = self.env.step(action)
|
||||
rewards.append(global_reward)
|
||||
if done:
|
||||
break
|
||||
ob = next_ob
|
||||
mean_reward = np.mean(np.array(rewards))
|
||||
std_reward = np.std(np.array(rewards))
|
||||
return mean_reward, std_reward
|
||||
|
||||
def run_thread(self, coord):
|
||||
'''Multi-threading is disabled'''
|
||||
ob = self.env.reset()
|
||||
done = False
|
||||
cum_reward = 0
|
||||
while not coord.should_stop():
|
||||
ob, done, R, cum_reward = self.explore(ob, done, cum_reward)
|
||||
global_step = self.global_counter.cur_step
|
||||
if self.agent.endswith('a2c'):
|
||||
self.model.backward(R, self.summary_writer, global_step)
|
||||
else:
|
||||
self.model.backward(self.summary_writer, global_step)
|
||||
self.summary_writer.flush()
|
||||
if (self.global_counter.should_stop()) and (not coord.should_stop()):
|
||||
self.env.terminate()
|
||||
coord.request_stop()
|
||||
logging.info('Training: stop condition reached!')
|
||||
return
|
||||
|
||||
def run(self):
|
||||
while not self.global_counter.should_stop():
|
||||
# test
|
||||
if self.run_test and self.global_counter.should_test():
|
||||
rewards = []
|
||||
global_step = self.global_counter.cur_step
|
||||
self.env.train_mode = False
|
||||
for test_ind in range(self.test_num):
|
||||
mean_reward, std_reward = self.perform(test_ind)
|
||||
self.env.terminate()
|
||||
rewards.append(mean_reward)
|
||||
log = {'agent': self.agent,
|
||||
'step': global_step,
|
||||
'test_id': test_ind,
|
||||
'avg_reward': mean_reward,
|
||||
'std_reward': std_reward}
|
||||
self.data.append(log)
|
||||
avg_reward = np.mean(np.array(rewards))
|
||||
self._add_summary(avg_reward, global_step, is_train=False)
|
||||
logging.info('Testing: global step %d, avg R: %.2f' %
|
||||
(global_step, avg_reward))
|
||||
# train
|
||||
self.env.train_mode = True
|
||||
ob = self.env.reset()
|
||||
# note this done is pre-decision to reset LSTM states!
|
||||
done = True
|
||||
self.model.reset()
|
||||
self.cur_step = 0
|
||||
rewards = []
|
||||
while True:
|
||||
ob, done, R, cur_rewards = self.explore(ob, done)
|
||||
rewards += cur_rewards
|
||||
global_step = self.global_counter.cur_step
|
||||
if self.agent.endswith('a2c'):
|
||||
self.model.backward(R, self.summary_writer, global_step)
|
||||
else:
|
||||
self.model.backward(self.summary_writer, global_step)
|
||||
# termination
|
||||
if done:
|
||||
# self.env.terminate()
|
||||
break
|
||||
rewards = np.array(rewards)
|
||||
mean_reward = np.mean(rewards)
|
||||
std_reward = np.std(rewards)
|
||||
log = {'agent': self.agent,
|
||||
'step': global_step,
|
||||
'test_id': -1,
|
||||
'avg_reward': mean_reward,
|
||||
'std_reward': std_reward}
|
||||
self.data.append(log)
|
||||
self._add_summary(mean_reward, global_step)
|
||||
self.summary_writer.flush()
|
||||
df = pd.DataFrame(self.data)
|
||||
df.to_csv(self.output_path + 'train_reward.csv')
|
||||
acts = np.array(self.acts, dtype=np.float32)
|
||||
Rs = np.array(self.Rs, dtype=np.float32)
|
||||
Advs = np.array(self.Advs, dtype=np.float32)
|
||||
# use pre-step dones here
|
||||
dones = np.array(self.dones[:-1], dtype=np.bool)
|
||||
self.reset(self.dones[-1])
|
||||
return obs, acts, dones, Rs, Advs
|
||||
|
||||
|
||||
class Tester(Trainer):
|
||||
def __init__(self, env, model, global_counter, summary_writer, output_path):
|
||||
super().__init__(env, model, global_counter, summary_writer)
|
||||
self.env.train_mode = False
|
||||
self.test_num = self.env.test_num
|
||||
self.output_path = output_path
|
||||
self.data = []
|
||||
logging.info('Testing: total test num: %d' % self.test_num)
|
||||
class ReplayBuffer(TransBuffer):
|
||||
def __init__(self, buffer_size, batch_size):
|
||||
self.buffer_size = buffer_size
|
||||
self.batch_size = batch_size
|
||||
self.cum_size = 0
|
||||
self.buffer = []
|
||||
|
||||
def _init_summary(self):
|
||||
self.reward = tf.placeholder(tensorflow.float32, [])
|
||||
self.summary = tf.summary.scalar('test_reward', self.reward)
|
||||
def add_transition(self, ob, a, r, next_ob, done):
|
||||
experience = (ob, a, r, next_ob, done)
|
||||
if self.cum_size < self.buffer_size:
|
||||
self.buffer.append(experience)
|
||||
else:
|
||||
ind = int(self.cum_size % self.buffer_size)
|
||||
self.buffer[ind] = experience
|
||||
self.cum_size += 1
|
||||
|
||||
def run_offline(self):
|
||||
# enable traffic measurments for offline test
|
||||
is_record = True
|
||||
record_stats = False
|
||||
self.env.cur_episode = 0
|
||||
self.env.init_data(is_record, record_stats, self.output_path)
|
||||
rewards = []
|
||||
for test_ind in range(self.test_num):
|
||||
rewards.append(self.perform(test_ind))
|
||||
self.env.terminate()
|
||||
time.sleep(2)
|
||||
self.env.collect_tripinfo()
|
||||
avg_reward = np.mean(np.array(rewards))
|
||||
logging.info('Offline testing: avg R: %.2f' % avg_reward)
|
||||
self.env.output_data()
|
||||
def reset(self):
|
||||
self.buffer = []
|
||||
self.cum_size = 0
|
||||
|
||||
def run_online(self, coord):
|
||||
self.env.cur_episode = 0
|
||||
while not coord.should_stop():
|
||||
time.sleep(30)
|
||||
if self.global_counter.should_test():
|
||||
rewards = []
|
||||
global_step = self.global_counter.cur_step
|
||||
for test_ind in range(self.test_num):
|
||||
cur_reward = self.perform(test_ind)
|
||||
self.env.terminate()
|
||||
rewards.append(cur_reward)
|
||||
log = {'agent': self.agent,
|
||||
'step': global_step,
|
||||
'test_id': test_ind,
|
||||
'reward': cur_reward}
|
||||
self.data.append(log)
|
||||
avg_reward = np.mean(np.array(rewards))
|
||||
self._add_summary(avg_reward, global_step)
|
||||
logging.info('Testing: global step %d, avg R: %.2f' %
|
||||
(global_step, avg_reward))
|
||||
# self.global_counter.update_test(avg_reward)
|
||||
df = pd.DataFrame(self.data)
|
||||
df.to_csv(self.output_path + 'train_reward.csv')
|
||||
def sample_transition(self):
|
||||
# Randomly sample batch_size examples
|
||||
minibatch = random.sample(self.buffer, self.batch_size)
|
||||
state_batch = np.asarray([data[0] for data in minibatch])
|
||||
action_batch = np.asarray([data[1] for data in minibatch])
|
||||
next_state_batch = np.asarray([data[3] for data in minibatch])
|
||||
reward_batch = np.asarray([data[2] for data in minibatch])
|
||||
done_batch = np.asarray([data[4] for data in minibatch])
|
||||
return state_batch, action_batch, next_state_batch, reward_batch, done_batch
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return min(self.buffer_size, self.cum_size)
|
||||
|
||||
|
||||
class Predictor(Tester):
|
||||
def __init__(self, env, model, demo=False, policy_type='default'):
|
||||
self.env = env
|
||||
self.model = model
|
||||
self.agent = self.env.agent
|
||||
self.env.train_mode = False
|
||||
self.test_num = self.env.test_num
|
||||
self.demo = demo
|
||||
self.policy_type = policy_type
|
||||
|
||||
def run(self, state):
|
||||
self.env.cur_episode = 0
|
||||
time.sleep(1)
|
||||
for test_ind in range(self.test_num):
|
||||
action = self.inference(state, policy_type=self.policy_type)
|
||||
time.sleep(2)
|
||||
|
||||
return action
|
||||
"""
|
||||
util functions
|
||||
"""
|
||||
|
||||
|
||||
class Evaluator(Tester):
|
||||
def __init__(self, env, model, output_path, demo=False, policy_type='default'):
|
||||
self.env = env
|
||||
self.model = model
|
||||
self.agent = self.env.agent
|
||||
self.env.train_mode = False
|
||||
self.test_num = self.env.test_num
|
||||
self.output_path = output_path
|
||||
self.demo = demo
|
||||
self.policy_type = policy_type
|
||||
class Scheduler:
|
||||
def __init__(self, val_init, val_min=0, total_step=0, decay='linear'):
|
||||
self.val = val_init
|
||||
self.N = float(total_step)
|
||||
self.val_min = val_min
|
||||
self.decay = decay
|
||||
self.n = 0
|
||||
|
||||
def run(self):
|
||||
is_record = True
|
||||
record_stats = False
|
||||
self.env.cur_episode = 0
|
||||
self.env.init_data(is_record, record_stats, self.output_path)
|
||||
time.sleep(1)
|
||||
for test_ind in range(self.test_num):
|
||||
reward, _ = self.perform(policy_type=self.policy_type)
|
||||
logging.info('test %i, avg reward %.2f' % (test_ind, reward))
|
||||
time.sleep(2)
|
||||
self.env.output_data()
|
||||
def get(self, n_step):
|
||||
self.n += n_step
|
||||
if self.decay == 'linear':
|
||||
return max(self.val_min, self.val * (1 - self.n / self.N))
|
||||
else:
|
||||
return self.val
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_layers()
|
||||
|
||||
Reference in New Issue
Block a user