205 KiB
205 KiB
In [1]:
import sys, os
sys.path.insert(0, os.path.abspath('..')) # Path fix
import os, json
from pathlib import Path
import numpy as np
import pandas as pd
from dotenv import load_dotenv
from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv
import gymnasium as gym
from gymnasium import spaces
import features # your features.py
load_dotenv()
SYMBOL = os.getenv("TRAINING_SYMBOL", "EURUSD")
TIMEFRAME = os.getenv("TIMEFRAME", "M15")
SPLIT_RATIO = float(os.getenv("SPLIT_RATIO", "0.8"))
DATA_CSV = Path("data") / f"ohlc_{SYMBOL}_{TIMEFRAME}.csv"
assert DATA_CSV.exists(), f"Missing dataset {DATA_CSV}. Run 1_Data.ipynb or provide CSV."
df = pd.read_csv(DATA_CSV, parse_dates=["time"], index_col="time")
df_feat = features.add_indicators(df.copy())
# --- ADD THIS LINE ---
df_feat.dropna(inplace=True)
# ---------------------
# Choose numeric columns as features (exclude obvious targets if any)
candidates = df_feat.select_dtypes(include=[np.number]).columns.tolist()
# Keep OHLCV + indicators for now
feature_cols = candidates
# Split
n_split = int(len(df_feat) * SPLIT_RATIO)
df_train = df_feat.iloc[:n_split].copy()
df_val = df_feat.iloc[n_split:].copy()
print("Train:", df_train.shape, "Val:", df_val.shape, "Features:", len(feature_cols))
Train: (39672, 17) Val: (9918, 17) Features: 17
In [2]:
# Minimal discrete trading environment (buy/hold/sell).
class TradingEnv(gym.Env):
metadata = {"render_modes": []}
def __init__(self, df_feat, feature_cols, trade_cost=1e-4):
super().__init__()
self.df = df_feat
self.cols = feature_cols
self.trade_cost = float(trade_cost)
self.n = len(self.df)
self.idx = 0
self.position = 0 # -1, 0, +1
self.observation_space = spaces.Box(
low=-np.inf, high=np.inf, shape=(len(self.cols),), dtype=np.float32
)
self.action_space = spaces.Discrete(3) # 0=sell, 1=hold, 2=buy
def _obs(self):
row = self.df.iloc[self.idx][self.cols].astype(float).values
return row.astype(np.float32)
def reset(self, seed=None, options=None):
super().reset(seed=seed)
self.idx = 1 # need a previous bar for return
self.position = 0
obs = self._obs()
return obs, {}
def step(self, action):
# map action -> position
pos_new = {-1:0, 0:0, 1:0, 2:1}[action] if action in (-1,0,1,2) else 0
pos_new = {-1: -1, 0: 0, 1: 1}.get({0:-1,1:0,2:1}[action], 0)
# price return from t-1 -> t on close
prev = self.df["close"].iloc[self.idx-1]
curr = self.df["close"].iloc[self.idx]
ret = (curr - prev) / (prev + 1e-12)
# reward is position * return minus cost if changed position
reward = pos_new * ret - (self.trade_cost if pos_new != self.position else 0.0)
self.position = pos_new
# next
terminated = False
self.idx += 1
truncated = self.idx >= (self.n - 1)
obs = self._obs()
info = {"position": self.position, "ret": ret}
return obs, float(reward), terminated, truncated, infoIn [4]:
# Train PPO
import numpy as np
from stable_baselines3.common.env_util import make_vec_env
def make_env():
return TradingEnv(df_train, feature_cols)
env = DummyVecEnv([make_env])
model = PPO("MlpPolicy", env, verbose=1)
total_timesteps = int(os.getenv("TOTAL_TIMESTEPS", "100000"))
model.learn(total_timesteps=total_timesteps)
Path("models").mkdir(exist_ok=True)
model_path = Path("models") / f"ppo_{SYMBOL}_{TIMEFRAME}.zip"
model.save(model_path.as_posix())
with open(Path("models") / "selected_features.json", "w", encoding="utf-8") as f:
json.dump(feature_cols, f, indent=2)
print("Saved model to:", model_path.resolve())Using cpu device ----------------------------- | time/ | | | fps | 710 | | iterations | 1 | | time_elapsed | 2 | | total_timesteps | 2048 | ----------------------------- ----------------------------------------- | time/ | | | fps | 587 | | iterations | 2 | | time_elapsed | 6 | | total_timesteps | 4096 | | train/ | | | approx_kl | 0.009429634 | | clip_fraction | 0.0859 | | clip_range | 0.2 | | entropy_loss | -1.09 | | explained_variance | -8.04 | | learning_rate | 0.0003 | | loss | -0.0214 | | n_updates | 10 | | policy_gradient_loss | -0.00715 | | value_loss | 0.00153 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 551 | | iterations | 3 | | time_elapsed | 11 | | total_timesteps | 6144 | | train/ | | | approx_kl | 0.0042742263 | | clip_fraction | 0.00928 | | clip_range | 0.2 | | entropy_loss | -1.09 | | explained_variance | -9.47 | | learning_rate | 0.0003 | | loss | -0.00427 | | n_updates | 20 | | policy_gradient_loss | -0.00337 | | value_loss | 0.000417 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 536 | | iterations | 4 | | time_elapsed | 15 | | total_timesteps | 8192 | | train/ | | | approx_kl | 0.0044627683 | | clip_fraction | 0.0359 | | clip_range | 0.2 | | entropy_loss | -1.06 | | explained_variance | -8.19 | | learning_rate | 0.0003 | | loss | -0.00498 | | n_updates | 30 | | policy_gradient_loss | -0.00413 | | value_loss | 0.000232 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 508 | | iterations | 5 | | time_elapsed | 20 | | total_timesteps | 10240 | | train/ | | | approx_kl | 0.0061096027 | | clip_fraction | 0.0265 | | clip_range | 0.2 | | entropy_loss | -1.03 | | explained_variance | -6.02 | | learning_rate | 0.0003 | | loss | -0.0174 | | n_updates | 40 | | policy_gradient_loss | -0.01 | | value_loss | 0.000184 | ------------------------------------------ ----------------------------------------- | time/ | | | fps | 503 | | iterations | 6 | | time_elapsed | 24 | | total_timesteps | 12288 | | train/ | | | approx_kl | 0.008018628 | | clip_fraction | 0.0893 | | clip_range | 0.2 | | entropy_loss | -1.03 | | explained_variance | -4.55 | | learning_rate | 0.0003 | | loss | -0.022 | | n_updates | 50 | | policy_gradient_loss | -0.00688 | | value_loss | 0.000197 | ----------------------------------------- ----------------------------------------- | time/ | | | fps | 505 | | iterations | 7 | | time_elapsed | 28 | | total_timesteps | 14336 | | train/ | | | approx_kl | 0.008238241 | | clip_fraction | 0.0252 | | clip_range | 0.2 | | entropy_loss | -1 | | explained_variance | -5 | | learning_rate | 0.0003 | | loss | -0.0208 | | n_updates | 60 | | policy_gradient_loss | -0.00627 | | value_loss | 7.93e-05 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 502 | | iterations | 8 | | time_elapsed | 32 | | total_timesteps | 16384 | | train/ | | | approx_kl | 0.0063306787 | | clip_fraction | 0.0549 | | clip_range | 0.2 | | entropy_loss | -0.952 | | explained_variance | -6.96 | | learning_rate | 0.0003 | | loss | -0.0044 | | n_updates | 70 | | policy_gradient_loss | -0.00798 | | value_loss | 7.12e-05 | ------------------------------------------ ---------------------------------------- | time/ | | | fps | 502 | | iterations | 9 | | time_elapsed | 36 | | total_timesteps | 18432 | | train/ | | | approx_kl | 0.00408049 | | clip_fraction | 0.0551 | | clip_range | 0.2 | | entropy_loss | -0.932 | | explained_variance | -2.84 | | learning_rate | 0.0003 | | loss | -0.017 | | n_updates | 80 | | policy_gradient_loss | -0.00532 | | value_loss | 4.31e-05 | ---------------------------------------- ----------------------------------------- | time/ | | | fps | 504 | | iterations | 10 | | time_elapsed | 40 | | total_timesteps | 20480 | | train/ | | | approx_kl | 0.008295992 | | clip_fraction | 0.0611 | | clip_range | 0.2 | | entropy_loss | -0.935 | | explained_variance | -8.56 | | learning_rate | 0.0003 | | loss | -0.0135 | | n_updates | 90 | | policy_gradient_loss | -0.00899 | | value_loss | 5.76e-05 | ----------------------------------------- ----------------------------------------- | time/ | | | fps | 506 | | iterations | 11 | | time_elapsed | 44 | | total_timesteps | 22528 | | train/ | | | approx_kl | 0.014076492 | | clip_fraction | 0.0576 | | clip_range | 0.2 | | entropy_loss | -0.879 | | explained_variance | -4.39 | | learning_rate | 0.0003 | | loss | 0.0102 | | n_updates | 100 | | policy_gradient_loss | -0.00607 | | value_loss | 2.66e-05 | ----------------------------------------- ----------------------------------------- | time/ | | | fps | 503 | | iterations | 12 | | time_elapsed | 48 | | total_timesteps | 24576 | | train/ | | | approx_kl | 0.008959841 | | clip_fraction | 0.068 | | clip_range | 0.2 | | entropy_loss | -0.912 | | explained_variance | -7.66 | | learning_rate | 0.0003 | | loss | -0.0189 | | n_updates | 110 | | policy_gradient_loss | -0.00744 | | value_loss | 2.75e-05 | ----------------------------------------- ----------------------------------------- | time/ | | | fps | 500 | | iterations | 13 | | time_elapsed | 53 | | total_timesteps | 26624 | | train/ | | | approx_kl | 0.008437004 | | clip_fraction | 0.0575 | | clip_range | 0.2 | | entropy_loss | -0.903 | | explained_variance | -4.35 | | learning_rate | 0.0003 | | loss | -0.0319 | | n_updates | 120 | | policy_gradient_loss | -0.00935 | | value_loss | 7.75e-05 | ----------------------------------------- --------------------------------------- | time/ | | | fps | 502 | | iterations | 14 | | time_elapsed | 57 | | total_timesteps | 28672 | | train/ | | | approx_kl | 0.0126811 | | clip_fraction | 0.119 | | clip_range | 0.2 | | entropy_loss | -0.816 | | explained_variance | -5.55 | | learning_rate | 0.0003 | | loss | -0.0323 | | n_updates | 130 | | policy_gradient_loss | -0.00794 | | value_loss | 0.000615 | --------------------------------------- ------------------------------------------ | time/ | | | fps | 503 | | iterations | 15 | | time_elapsed | 61 | | total_timesteps | 30720 | | train/ | | | approx_kl | 0.0075738453 | | clip_fraction | 0.0583 | | clip_range | 0.2 | | entropy_loss | -0.722 | | explained_variance | -10.5 | | learning_rate | 0.0003 | | loss | -0.0262 | | n_updates | 140 | | policy_gradient_loss | -0.00878 | | value_loss | 1.59e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 504 | | iterations | 16 | | time_elapsed | 65 | | total_timesteps | 32768 | | train/ | | | approx_kl | 0.0042516133 | | clip_fraction | 0.0463 | | clip_range | 0.2 | | entropy_loss | -0.624 | | explained_variance | -16.7 | | learning_rate | 0.0003 | | loss | -0.0243 | | n_updates | 150 | | policy_gradient_loss | -0.00972 | | value_loss | 1.29e-05 | ------------------------------------------ ---------------------------------------- | time/ | | | fps | 504 | | iterations | 17 | | time_elapsed | 68 | | total_timesteps | 34816 | | train/ | | | approx_kl | 0.00597096 | | clip_fraction | 0.0776 | | clip_range | 0.2 | | entropy_loss | -0.544 | | explained_variance | -5.51 | | learning_rate | 0.0003 | | loss | -0.00566 | | n_updates | 160 | | policy_gradient_loss | -0.00572 | | value_loss | 1.4e-05 | ---------------------------------------- ----------------------------------------- | time/ | | | fps | 504 | | iterations | 18 | | time_elapsed | 73 | | total_timesteps | 36864 | | train/ | | | approx_kl | 0.005309558 | | clip_fraction | 0.0558 | | clip_range | 0.2 | | entropy_loss | -0.525 | | explained_variance | -9.82 | | learning_rate | 0.0003 | | loss | -0.00756 | | n_updates | 170 | | policy_gradient_loss | -0.00672 | | value_loss | 5.88e-05 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 505 | | iterations | 19 | | time_elapsed | 76 | | total_timesteps | 38912 | | train/ | | | approx_kl | 0.0060142255 | | clip_fraction | 0.0741 | | clip_range | 0.2 | | entropy_loss | -0.477 | | explained_variance | -3.59 | | learning_rate | 0.0003 | | loss | -0.0209 | | n_updates | 180 | | policy_gradient_loss | -0.00457 | | value_loss | 1.85e-05 | ------------------------------------------ ---------------------------------------- | time/ | | | fps | 504 | | iterations | 20 | | time_elapsed | 81 | | total_timesteps | 40960 | | train/ | | | approx_kl | 0.00463128 | | clip_fraction | 0.0469 | | clip_range | 0.2 | | entropy_loss | -0.407 | | explained_variance | -4.95 | | learning_rate | 0.0003 | | loss | -0.0257 | | n_updates | 190 | | policy_gradient_loss | -0.00473 | | value_loss | 0.00014 | ---------------------------------------- ------------------------------------------ | time/ | | | fps | 504 | | iterations | 21 | | time_elapsed | 85 | | total_timesteps | 43008 | | train/ | | | approx_kl | 0.0023931228 | | clip_fraction | 0.0385 | | clip_range | 0.2 | | entropy_loss | -0.414 | | explained_variance | -4.5 | | learning_rate | 0.0003 | | loss | -0.0209 | | n_updates | 200 | | policy_gradient_loss | -0.005 | | value_loss | 9.36e-06 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 505 | | iterations | 22 | | time_elapsed | 89 | | total_timesteps | 45056 | | train/ | | | approx_kl | 0.0034446488 | | clip_fraction | 0.0373 | | clip_range | 0.2 | | entropy_loss | -0.354 | | explained_variance | -12.8 | | learning_rate | 0.0003 | | loss | -0.0109 | | n_updates | 210 | | policy_gradient_loss | -0.0059 | | value_loss | 7.07e-05 | ------------------------------------------ ----------------------------------------- | time/ | | | fps | 506 | | iterations | 23 | | time_elapsed | 93 | | total_timesteps | 47104 | | train/ | | | approx_kl | 0.002063186 | | clip_fraction | 0.0291 | | clip_range | 0.2 | | entropy_loss | -0.302 | | explained_variance | -5.94 | | learning_rate | 0.0003 | | loss | 0.000529 | | n_updates | 220 | | policy_gradient_loss | -0.006 | | value_loss | 2.82e-05 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 505 | | iterations | 24 | | time_elapsed | 97 | | total_timesteps | 49152 | | train/ | | | approx_kl | 0.0032632346 | | clip_fraction | 0.0346 | | clip_range | 0.2 | | entropy_loss | -0.364 | | explained_variance | -5.95 | | learning_rate | 0.0003 | | loss | -0.0142 | | n_updates | 230 | | policy_gradient_loss | -0.00837 | | value_loss | 1.7e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 506 | | iterations | 25 | | time_elapsed | 101 | | total_timesteps | 51200 | | train/ | | | approx_kl | 0.0054131867 | | clip_fraction | 0.0503 | | clip_range | 0.2 | | entropy_loss | -0.303 | | explained_variance | -5.86 | | learning_rate | 0.0003 | | loss | -0.0238 | | n_updates | 240 | | policy_gradient_loss | -0.00928 | | value_loss | 1.5e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 505 | | iterations | 26 | | time_elapsed | 105 | | total_timesteps | 53248 | | train/ | | | approx_kl | 0.0050631175 | | clip_fraction | 0.0424 | | clip_range | 0.2 | | entropy_loss | -0.313 | | explained_variance | -7.31 | | learning_rate | 0.0003 | | loss | 0.0182 | | n_updates | 250 | | policy_gradient_loss | -0.00901 | | value_loss | 0.000127 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 505 | | iterations | 27 | | time_elapsed | 109 | | total_timesteps | 55296 | | train/ | | | approx_kl | 0.0026903055 | | clip_fraction | 0.0326 | | clip_range | 0.2 | | entropy_loss | -0.291 | | explained_variance | -10 | | learning_rate | 0.0003 | | loss | 0.00649 | | n_updates | 260 | | policy_gradient_loss | -0.00568 | | value_loss | 2.96e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 505 | | iterations | 28 | | time_elapsed | 113 | | total_timesteps | 57344 | | train/ | | | approx_kl | 0.0036246267 | | clip_fraction | 0.0355 | | clip_range | 0.2 | | entropy_loss | -0.271 | | explained_variance | -9.29 | | learning_rate | 0.0003 | | loss | -0.00148 | | n_updates | 270 | | policy_gradient_loss | -0.00581 | | value_loss | 4.05e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 505 | | iterations | 29 | | time_elapsed | 117 | | total_timesteps | 59392 | | train/ | | | approx_kl | 0.0024851589 | | clip_fraction | 0.0375 | | clip_range | 0.2 | | entropy_loss | -0.284 | | explained_variance | -5.18 | | learning_rate | 0.0003 | | loss | 0.0342 | | n_updates | 280 | | policy_gradient_loss | -0.00556 | | value_loss | 1.29e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 499 | | iterations | 30 | | time_elapsed | 122 | | total_timesteps | 61440 | | train/ | | | approx_kl | 0.0028119227 | | clip_fraction | 0.0378 | | clip_range | 0.2 | | entropy_loss | -0.276 | | explained_variance | -3.17 | | learning_rate | 0.0003 | | loss | -0.0172 | | n_updates | 290 | | policy_gradient_loss | -0.00783 | | value_loss | 3.1e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 484 | | iterations | 31 | | time_elapsed | 131 | | total_timesteps | 63488 | | train/ | | | approx_kl | 0.0064774947 | | clip_fraction | 0.0411 | | clip_range | 0.2 | | entropy_loss | -0.249 | | explained_variance | -8.33 | | learning_rate | 0.0003 | | loss | -0.0163 | | n_updates | 300 | | policy_gradient_loss | -0.00733 | | value_loss | 5.18e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 472 | | iterations | 32 | | time_elapsed | 138 | | total_timesteps | 65536 | | train/ | | | approx_kl | 0.0043396214 | | clip_fraction | 0.0306 | | clip_range | 0.2 | | entropy_loss | -0.258 | | explained_variance | -5 | | learning_rate | 0.0003 | | loss | -0.0161 | | n_updates | 310 | | policy_gradient_loss | -0.00749 | | value_loss | 0.000281 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 467 | | iterations | 33 | | time_elapsed | 144 | | total_timesteps | 67584 | | train/ | | | approx_kl | 0.0057827905 | | clip_fraction | 0.0433 | | clip_range | 0.2 | | entropy_loss | -0.239 | | explained_variance | -9 | | learning_rate | 0.0003 | | loss | 0.0578 | | n_updates | 320 | | policy_gradient_loss | -0.0075 | | value_loss | 8.34e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 451 | | iterations | 34 | | time_elapsed | 154 | | total_timesteps | 69632 | | train/ | | | approx_kl | 0.0021125488 | | clip_fraction | 0.0399 | | clip_range | 0.2 | | entropy_loss | -0.182 | | explained_variance | -3.59 | | learning_rate | 0.0003 | | loss | -0.0116 | | n_updates | 330 | | policy_gradient_loss | -0.00369 | | value_loss | 1.17e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 441 | | iterations | 35 | | time_elapsed | 162 | | total_timesteps | 71680 | | train/ | | | approx_kl | 0.0020559407 | | clip_fraction | 0.0153 | | clip_range | 0.2 | | entropy_loss | -0.19 | | explained_variance | -3.77 | | learning_rate | 0.0003 | | loss | -0.00623 | | n_updates | 340 | | policy_gradient_loss | -0.00447 | | value_loss | 2.41e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 432 | | iterations | 36 | | time_elapsed | 170 | | total_timesteps | 73728 | | train/ | | | approx_kl | 0.0029917397 | | clip_fraction | 0.0263 | | clip_range | 0.2 | | entropy_loss | -0.178 | | explained_variance | -4.57 | | learning_rate | 0.0003 | | loss | 0.0217 | | n_updates | 350 | | policy_gradient_loss | -0.0037 | | value_loss | 2.88e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 426 | | iterations | 37 | | time_elapsed | 177 | | total_timesteps | 75776 | | train/ | | | approx_kl | 0.0024150917 | | clip_fraction | 0.0241 | | clip_range | 0.2 | | entropy_loss | -0.138 | | explained_variance | -4.13 | | learning_rate | 0.0003 | | loss | -0.0247 | | n_updates | 360 | | policy_gradient_loss | -0.00457 | | value_loss | 5.44e-05 | ------------------------------------------ ------------------------------------------- | time/ | | | fps | 420 | | iterations | 38 | | time_elapsed | 185 | | total_timesteps | 77824 | | train/ | | | approx_kl | 0.00088574155 | | clip_fraction | 0.0103 | | clip_range | 0.2 | | entropy_loss | -0.102 | | explained_variance | -2.32 | | learning_rate | 0.0003 | | loss | 0.0483 | | n_updates | 370 | | policy_gradient_loss | -0.00162 | | value_loss | 9.93e-05 | ------------------------------------------- ------------------------------------------ | time/ | | | fps | 410 | | iterations | 39 | | time_elapsed | 194 | | total_timesteps | 79872 | | train/ | | | approx_kl | 0.0015186302 | | clip_fraction | 0.0154 | | clip_range | 0.2 | | entropy_loss | -0.115 | | explained_variance | -4.36 | | learning_rate | 0.0003 | | loss | 0.00566 | | n_updates | 380 | | policy_gradient_loss | -0.00212 | | value_loss | 1.36e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 404 | | iterations | 40 | | time_elapsed | 202 | | total_timesteps | 81920 | | train/ | | | approx_kl | 0.0010117381 | | clip_fraction | 0.0147 | | clip_range | 0.2 | | entropy_loss | -0.0774 | | explained_variance | -13.6 | | learning_rate | 0.0003 | | loss | 0.0186 | | n_updates | 390 | | policy_gradient_loss | -0.00316 | | value_loss | 1.99e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 400 | | iterations | 41 | | time_elapsed | 209 | | total_timesteps | 83968 | | train/ | | | approx_kl | 0.0011792822 | | clip_fraction | 0.013 | | clip_range | 0.2 | | entropy_loss | -0.103 | | explained_variance | -2.33 | | learning_rate | 0.0003 | | loss | 8.65e-05 | | n_updates | 400 | | policy_gradient_loss | -0.00386 | | value_loss | 9.81e-05 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 395 | | iterations | 42 | | time_elapsed | 217 | | total_timesteps | 86016 | | train/ | | | approx_kl | 0.0017598666 | | clip_fraction | 0.0228 | | clip_range | 0.2 | | entropy_loss | -0.103 | | explained_variance | -6.97 | | learning_rate | 0.0003 | | loss | -0.0242 | | n_updates | 410 | | policy_gradient_loss | -0.00448 | | value_loss | 2.37e-05 | ------------------------------------------ ---------------------------------------- | time/ | | | fps | 388 | | iterations | 43 | | time_elapsed | 226 | | total_timesteps | 88064 | | train/ | | | approx_kl | 0.00175269 | | clip_fraction | 0.0197 | | clip_range | 0.2 | | entropy_loss | -0.144 | | explained_variance | -1.69 | | learning_rate | 0.0003 | | loss | -0.0162 | | n_updates | 420 | | policy_gradient_loss | -0.00368 | | value_loss | 9.39e-06 | ---------------------------------------- ----------------------------------------- | time/ | | | fps | 381 | | iterations | 44 | | time_elapsed | 236 | | total_timesteps | 90112 | | train/ | | | approx_kl | 0.003029982 | | clip_fraction | 0.0339 | | clip_range | 0.2 | | entropy_loss | -0.162 | | explained_variance | -2.24 | | learning_rate | 0.0003 | | loss | -0.0335 | | n_updates | 430 | | policy_gradient_loss | -0.00511 | | value_loss | 8.37e-06 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 380 | | iterations | 45 | | time_elapsed | 242 | | total_timesteps | 92160 | | train/ | | | approx_kl | 0.0027306322 | | clip_fraction | 0.0239 | | clip_range | 0.2 | | entropy_loss | -0.145 | | explained_variance | -3.44 | | learning_rate | 0.0003 | | loss | -0.008 | | n_updates | 440 | | policy_gradient_loss | -0.00584 | | value_loss | 1.1e-05 | ------------------------------------------ ----------------------------------------- | time/ | | | fps | 375 | | iterations | 46 | | time_elapsed | 250 | | total_timesteps | 94208 | | train/ | | | approx_kl | 0.003130577 | | clip_fraction | 0.0269 | | clip_range | 0.2 | | entropy_loss | -0.162 | | explained_variance | -3.31 | | learning_rate | 0.0003 | | loss | -0.0218 | | n_updates | 450 | | policy_gradient_loss | -0.00694 | | value_loss | 0.000233 | ----------------------------------------- ----------------------------------------- | time/ | | | fps | 371 | | iterations | 47 | | time_elapsed | 259 | | total_timesteps | 96256 | | train/ | | | approx_kl | 0.005973259 | | clip_fraction | 0.044 | | clip_range | 0.2 | | entropy_loss | -0.145 | | explained_variance | -6.63 | | learning_rate | 0.0003 | | loss | 0.0204 | | n_updates | 460 | | policy_gradient_loss | -0.0145 | | value_loss | 1.8e-05 | ----------------------------------------- ------------------------------------------ | time/ | | | fps | 368 | | iterations | 48 | | time_elapsed | 267 | | total_timesteps | 98304 | | train/ | | | approx_kl | 0.0028656125 | | clip_fraction | 0.0293 | | clip_range | 0.2 | | entropy_loss | -0.152 | | explained_variance | -2.07 | | learning_rate | 0.0003 | | loss | -0.0346 | | n_updates | 470 | | policy_gradient_loss | -0.00659 | | value_loss | 7.13e-06 | ------------------------------------------ ------------------------------------------ | time/ | | | fps | 368 | | iterations | 49 | | time_elapsed | 272 | | total_timesteps | 100352 | | train/ | | | approx_kl | 0.0018704929 | | clip_fraction | 0.0213 | | clip_range | 0.2 | | entropy_loss | -0.14 | | explained_variance | -3.72 | | learning_rate | 0.0003 | | loss | -0.00298 | | n_updates | 480 | | policy_gradient_loss | -0.00328 | | value_loss | 1.33e-05 | ------------------------------------------ Saved model to: G:\My Drive\Bots DRL\DRL\DRL-MT5-Lab\notebooks\models\ppo_EURUSD_M15.zip
In [3]:
# === Train A2C (same env/data as PPO) ===
import os, json
from pathlib import Path
from stable_baselines3 import A2C
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3.common.monitor import Monitor
def make_env():
return Monitor(TradingEnv(df_train, feature_cols))
env_a2c = DummyVecEnv([make_env])
a2c = A2C("MlpPolicy", env_a2c,
verbose=1,
n_steps=5, # SB3 default
gamma=0.99,
learning_rate=7e-4)
total_timesteps = int(os.getenv("TOTAL_TIMESTEPS", "100000"))
a2c.learn(total_timesteps=total_timesteps)
Path("models").mkdir(exist_ok=True)
a2c_path = Path("models") / f"a2c_{SYMBOL}_{TIMEFRAME}.zip"
a2c.save(a2c_path.as_posix())
# (Write once is enough; safe to overwrite)
with open(Path("models") / "selected_features.json", "w", encoding="utf-8") as f:
json.dump(feature_cols, f, indent=2)
print("Saved A2C model to:", a2c_path.resolve())
Using cpu device ------------------------------------ | time/ | | | fps | 359 | | iterations | 100 | | time_elapsed | 1 | | total_timesteps | 500 | | train/ | | | entropy_loss | -1.05 | | explained_variance | -12.1 | | learning_rate | 0.0007 | | n_updates | 99 | | policy_loss | 0.00607 | | value_loss | 5.1e-05 | ------------------------------------ ------------------------------------ | time/ | | | fps | 362 | | iterations | 200 | | time_elapsed | 2 | | total_timesteps | 1000 | | train/ | | | entropy_loss | -1.05 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 199 | | policy_loss | -0.00202 | | value_loss | 3.28e-06 | ------------------------------------ ------------------------------------- | time/ | | | fps | 370 | | iterations | 300 | | time_elapsed | 4 | | total_timesteps | 1500 | | train/ | | | entropy_loss | -1.02 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 299 | | policy_loss | -0.000347 | | value_loss | 1.86e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 352 | | iterations | 400 | | time_elapsed | 5 | | total_timesteps | 2000 | | train/ | | | entropy_loss | -0.896 | | explained_variance | 0.0136 | | learning_rate | 0.0007 | | n_updates | 399 | | policy_loss | 0.00511 | | value_loss | 2.97e-05 | ------------------------------------ ------------------------------------ | time/ | | | fps | 345 | | iterations | 500 | | time_elapsed | 7 | | total_timesteps | 2500 | | train/ | | | entropy_loss | -1.02 | | explained_variance | -1.63 | | learning_rate | 0.0007 | | n_updates | 499 | | policy_loss | 0.0098 | | value_loss | 0.000109 | ------------------------------------ ------------------------------------ | time/ | | | fps | 341 | | iterations | 600 | | time_elapsed | 8 | | total_timesteps | 3000 | | train/ | | | entropy_loss | -1.04 | | explained_variance | -64.9 | | learning_rate | 0.0007 | | n_updates | 599 | | policy_loss | 0.00834 | | value_loss | 0.000233 | ------------------------------------ ------------------------------------ | time/ | | | fps | 331 | | iterations | 700 | | time_elapsed | 10 | | total_timesteps | 3500 | | train/ | | | entropy_loss | -0.92 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 699 | | policy_loss | 0.000506 | | value_loss | 2.37e-07 | ------------------------------------ ------------------------------------ | time/ | | | fps | 328 | | iterations | 800 | | time_elapsed | 12 | | total_timesteps | 4000 | | train/ | | | entropy_loss | -0.941 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 799 | | policy_loss | 0.000532 | | value_loss | 3.67e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 326 | | iterations | 900 | | time_elapsed | 13 | | total_timesteps | 4500 | | train/ | | | entropy_loss | -0.851 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 899 | | policy_loss | -0.000172 | | value_loss | 2.84e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 324 | | iterations | 1000 | | time_elapsed | 15 | | total_timesteps | 5000 | | train/ | | | entropy_loss | -0.599 | | explained_variance | -4.26 | | learning_rate | 0.0007 | | n_updates | 999 | | policy_loss | 0.00652 | | value_loss | 7.48e-05 | ------------------------------------ ------------------------------------ | time/ | | | fps | 321 | | iterations | 1100 | | time_elapsed | 17 | | total_timesteps | 5500 | | train/ | | | entropy_loss | -0.598 | | explained_variance | -67.3 | | learning_rate | 0.0007 | | n_updates | 1099 | | policy_loss | -0.00226 | | value_loss | 5.66e-05 | ------------------------------------ ------------------------------------- | time/ | | | fps | 320 | | iterations | 1200 | | time_elapsed | 18 | | total_timesteps | 6000 | | train/ | | | entropy_loss | -0.6 | | explained_variance | -0.0421 | | learning_rate | 0.0007 | | n_updates | 1199 | | policy_loss | -0.000824 | | value_loss | 6.37e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 322 | | iterations | 1300 | | time_elapsed | 20 | | total_timesteps | 6500 | | train/ | | | entropy_loss | -0.96 | | explained_variance | -8.23e+03 | | learning_rate | 0.0007 | | n_updates | 1299 | | policy_loss | -0.0152 | | value_loss | 0.00129 | ------------------------------------- ------------------------------------- | time/ | | | fps | 326 | | iterations | 1400 | | time_elapsed | 21 | | total_timesteps | 7000 | | train/ | | | entropy_loss | -1 | | explained_variance | -1.37e+03 | | learning_rate | 0.0007 | | n_updates | 1399 | | policy_loss | 0.00168 | | value_loss | 1.88e-05 | ------------------------------------- ------------------------------------ | time/ | | | fps | 328 | | iterations | 1500 | | time_elapsed | 22 | | total_timesteps | 7500 | | train/ | | | entropy_loss | -0.737 | | explained_variance | -3.79 | | learning_rate | 0.0007 | | n_updates | 1499 | | policy_loss | 0.00252 | | value_loss | 1.9e-05 | ------------------------------------ ------------------------------------ | time/ | | | fps | 330 | | iterations | 1600 | | time_elapsed | 24 | | total_timesteps | 8000 | | train/ | | | entropy_loss | -0.657 | | explained_variance | -43.7 | | learning_rate | 0.0007 | | n_updates | 1599 | | policy_loss | 0.0008 | | value_loss | 1.34e-05 | ------------------------------------ ------------------------------------- | time/ | | | fps | 332 | | iterations | 1700 | | time_elapsed | 25 | | total_timesteps | 8500 | | train/ | | | entropy_loss | -0.63 | | explained_variance | -7.88 | | learning_rate | 0.0007 | | n_updates | 1699 | | policy_loss | -0.000272 | | value_loss | 1.75e-06 | ------------------------------------- ------------------------------------ | time/ | | | fps | 332 | | iterations | 1800 | | time_elapsed | 27 | | total_timesteps | 9000 | | train/ | | | entropy_loss | -0.531 | | explained_variance | -159 | | learning_rate | 0.0007 | | n_updates | 1799 | | policy_loss | -0.00301 | | value_loss | 0.000184 | ------------------------------------ ------------------------------------- | time/ | | | fps | 331 | | iterations | 1900 | | time_elapsed | 28 | | total_timesteps | 9500 | | train/ | | | entropy_loss | -0.561 | | explained_variance | -3.84e+05 | | learning_rate | 0.0007 | | n_updates | 1899 | | policy_loss | 0.00499 | | value_loss | 0.0105 | ------------------------------------- ------------------------------------ | time/ | | | fps | 329 | | iterations | 2000 | | time_elapsed | 30 | | total_timesteps | 10000 | | train/ | | | entropy_loss | -0.377 | | explained_variance | -894 | | learning_rate | 0.0007 | | n_updates | 1999 | | policy_loss | -0.00021 | | value_loss | 8.1e-06 | ------------------------------------ ------------------------------------- | time/ | | | fps | 328 | | iterations | 2100 | | time_elapsed | 31 | | total_timesteps | 10500 | | train/ | | | entropy_loss | -0.357 | | explained_variance | 0.00573 | | learning_rate | 0.0007 | | n_updates | 2099 | | policy_loss | -0.000193 | | value_loss | 4.08e-06 | ------------------------------------- ------------------------------------ | time/ | | | fps | 328 | | iterations | 2200 | | time_elapsed | 33 | | total_timesteps | 11000 | | train/ | | | entropy_loss | -0.196 | | explained_variance | -57 | | learning_rate | 0.0007 | | n_updates | 2199 | | policy_loss | 0.000124 | | value_loss | 8.03e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 326 | | iterations | 2300 | | time_elapsed | 35 | | total_timesteps | 11500 | | train/ | | | entropy_loss | -0.0896 | | explained_variance | -139 | | learning_rate | 0.0007 | | n_updates | 2299 | | policy_loss | 2.65e-05 | | value_loss | 6.23e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 324 | | iterations | 2400 | | time_elapsed | 36 | | total_timesteps | 12000 | | train/ | | | entropy_loss | -0.302 | | explained_variance | -58.3 | | learning_rate | 0.0007 | | n_updates | 2399 | | policy_loss | 0.000408 | | value_loss | 7.1e-07 | ------------------------------------ ------------------------------------ | time/ | | | fps | 324 | | iterations | 2500 | | time_elapsed | 38 | | total_timesteps | 12500 | | train/ | | | entropy_loss | -0.0701 | | explained_variance | 0.172 | | learning_rate | 0.0007 | | n_updates | 2499 | | policy_loss | 3.23e-08 | | value_loss | 1.81e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 323 | | iterations | 2600 | | time_elapsed | 40 | | total_timesteps | 13000 | | train/ | | | entropy_loss | -0.0533 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 2599 | | policy_loss | -9.45e-07 | | value_loss | 4.53e-08 | ------------------------------------- ------------------------------------ | time/ | | | fps | 322 | | iterations | 2700 | | time_elapsed | 41 | | total_timesteps | 13500 | | train/ | | | entropy_loss | -0.0479 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 2699 | | policy_loss | 4.05e-06 | | value_loss | 3.2e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 321 | | iterations | 2800 | | time_elapsed | 43 | | total_timesteps | 14000 | | train/ | | | entropy_loss | -0.0477 | | explained_variance | -0.807 | | learning_rate | 0.0007 | | n_updates | 2799 | | policy_loss | -9.65e-07 | | value_loss | 3.52e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 320 | | iterations | 2900 | | time_elapsed | 45 | | total_timesteps | 14500 | | train/ | | | entropy_loss | -0.141 | | explained_variance | -33.4 | | learning_rate | 0.0007 | | n_updates | 2899 | | policy_loss | -9.53e-05 | | value_loss | 5.38e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 319 | | iterations | 3000 | | time_elapsed | 46 | | total_timesteps | 15000 | | train/ | | | entropy_loss | -0.0618 | | explained_variance | -0.102 | | learning_rate | 0.0007 | | n_updates | 2999 | | policy_loss | -2.49e-06 | | value_loss | 7.56e-08 | ------------------------------------- ------------------------------------- | time/ | | | fps | 319 | | iterations | 3100 | | time_elapsed | 48 | | total_timesteps | 15500 | | train/ | | | entropy_loss | -0.0417 | | explained_variance | -0.0284 | | learning_rate | 0.0007 | | n_updates | 3099 | | policy_loss | -9.97e-06 | | value_loss | 3.2e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 319 | | iterations | 3200 | | time_elapsed | 50 | | total_timesteps | 16000 | | train/ | | | entropy_loss | -0.0643 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 3199 | | policy_loss | 1.16e-05 | | value_loss | 1.58e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 321 | | iterations | 3300 | | time_elapsed | 51 | | total_timesteps | 16500 | | train/ | | | entropy_loss | -0.061 | | explained_variance | -6.01 | | learning_rate | 0.0007 | | n_updates | 3299 | | policy_loss | -2.23e-05 | | value_loss | 5.67e-06 | ------------------------------------- ------------------------------------ | time/ | | | fps | 322 | | iterations | 3400 | | time_elapsed | 52 | | total_timesteps | 17000 | | train/ | | | entropy_loss | -0.0524 | | explained_variance | 0.349 | | learning_rate | 0.0007 | | n_updates | 3399 | | policy_loss | 4.4e-05 | | value_loss | 2.86e-05 | ------------------------------------ ------------------------------------- | time/ | | | fps | 323 | | iterations | 3500 | | time_elapsed | 54 | | total_timesteps | 17500 | | train/ | | | entropy_loss | -0.0491 | | explained_variance | -30.4 | | learning_rate | 0.0007 | | n_updates | 3499 | | policy_loss | -2.74e-05 | | value_loss | 1.57e-05 | ------------------------------------- ------------------------------------ | time/ | | | fps | 324 | | iterations | 3600 | | time_elapsed | 55 | | total_timesteps | 18000 | | train/ | | | entropy_loss | -0.0216 | | explained_variance | 0.00218 | | learning_rate | 0.0007 | | n_updates | 3599 | | policy_loss | -1.1e-06 | | value_loss | 2.05e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 324 | | iterations | 3700 | | time_elapsed | 57 | | total_timesteps | 18500 | | train/ | | | entropy_loss | -0.0218 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 3699 | | policy_loss | -5.02e-07 | | value_loss | 3.1e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 323 | | iterations | 3800 | | time_elapsed | 58 | | total_timesteps | 19000 | | train/ | | | entropy_loss | -0.024 | | explained_variance | -4.98 | | learning_rate | 0.0007 | | n_updates | 3799 | | policy_loss | -5.68e-06 | | value_loss | 4.92e-06 | ------------------------------------- ------------------------------------ | time/ | | | fps | 322 | | iterations | 3900 | | time_elapsed | 60 | | total_timesteps | 19500 | | train/ | | | entropy_loss | -0.0226 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 3899 | | policy_loss | 4.01e-07 | | value_loss | 6.52e-08 | ------------------------------------ ------------------------------------- | time/ | | | fps | 322 | | iterations | 4000 | | time_elapsed | 62 | | total_timesteps | 20000 | | train/ | | | entropy_loss | -0.0181 | | explained_variance | -1.72e-05 | | learning_rate | 0.0007 | | n_updates | 3999 | | policy_loss | 1.36e-06 | | value_loss | 3.18e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 322 | | iterations | 4100 | | time_elapsed | 63 | | total_timesteps | 20500 | | train/ | | | entropy_loss | -0.0168 | | explained_variance | 5.96e-08 | | learning_rate | 0.0007 | | n_updates | 4099 | | policy_loss | 1.98e-06 | | value_loss | 8.21e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 322 | | iterations | 4200 | | time_elapsed | 65 | | total_timesteps | 21000 | | train/ | | | entropy_loss | -0.0137 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 4199 | | policy_loss | -1.08e-06 | | value_loss | 5.07e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 322 | | iterations | 4300 | | time_elapsed | 66 | | total_timesteps | 21500 | | train/ | | | entropy_loss | -0.0155 | | explained_variance | -24.2 | | learning_rate | 0.0007 | | n_updates | 4299 | | policy_loss | 0.00232 | | value_loss | 3.59e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 321 | | iterations | 4400 | | time_elapsed | 68 | | total_timesteps | 22000 | | train/ | | | entropy_loss | -0.0163 | | explained_variance | -1.56 | | learning_rate | 0.0007 | | n_updates | 4399 | | policy_loss | 5.63e-07 | | value_loss | 1.49e-07 | ------------------------------------ ------------------------------------ | time/ | | | fps | 321 | | iterations | 4500 | | time_elapsed | 70 | | total_timesteps | 22500 | | train/ | | | entropy_loss | -0.0176 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 4499 | | policy_loss | 7.63e-07 | | value_loss | 2.56e-07 | ------------------------------------ ------------------------------------ | time/ | | | fps | 320 | | iterations | 4600 | | time_elapsed | 71 | | total_timesteps | 23000 | | train/ | | | entropy_loss | -0.0185 | | explained_variance | 5.96e-08 | | learning_rate | 0.0007 | | n_updates | 4599 | | policy_loss | 3.89e-06 | | value_loss | 2.59e-06 | ------------------------------------ ------------------------------------- | time/ | | | fps | 321 | | iterations | 4700 | | time_elapsed | 73 | | total_timesteps | 23500 | | train/ | | | entropy_loss | -0.0184 | | explained_variance | 5.96e-08 | | learning_rate | 0.0007 | | n_updates | 4699 | | policy_loss | -5.51e-07 | | value_loss | 2.17e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 320 | | iterations | 4800 | | time_elapsed | 74 | | total_timesteps | 24000 | | train/ | | | entropy_loss | -0.0183 | | explained_variance | -5.05 | | learning_rate | 0.0007 | | n_updates | 4799 | | policy_loss | -1.53e-06 | | value_loss | 6.76e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 320 | | iterations | 4900 | | time_elapsed | 76 | | total_timesteps | 24500 | | train/ | | | entropy_loss | -0.016 | | explained_variance | -2.41 | | learning_rate | 0.0007 | | n_updates | 4899 | | policy_loss | -1.9e-06 | | value_loss | 2.45e-06 | ------------------------------------ ------------------------------------- | time/ | | | fps | 319 | | iterations | 5000 | | time_elapsed | 78 | | total_timesteps | 25000 | | train/ | | | entropy_loss | -0.02 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 4999 | | policy_loss | -1.46e-06 | | value_loss | 6.88e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 320 | | iterations | 5100 | | time_elapsed | 79 | | total_timesteps | 25500 | | train/ | | | entropy_loss | -0.021 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 5099 | | policy_loss | -3.64e-06 | | value_loss | 1.73e-06 | ------------------------------------- ------------------------------------ | time/ | | | fps | 321 | | iterations | 5200 | | time_elapsed | 80 | | total_timesteps | 26000 | | train/ | | | entropy_loss | -0.0193 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 5199 | | policy_loss | 6.82e-07 | | value_loss | 9.1e-08 | ------------------------------------ ------------------------------------ | time/ | | | fps | 322 | | iterations | 5300 | | time_elapsed | 82 | | total_timesteps | 26500 | | train/ | | | entropy_loss | -0.019 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 5299 | | policy_loss | 6.54e-07 | | value_loss | 9.62e-08 | ------------------------------------ ------------------------------------- | time/ | | | fps | 323 | | iterations | 5400 | | time_elapsed | 83 | | total_timesteps | 27000 | | train/ | | | entropy_loss | -0.0312 | | explained_variance | 2.99e-05 | | learning_rate | 0.0007 | | n_updates | 5399 | | policy_loss | -1.18e-06 | | value_loss | 9.25e-08 | ------------------------------------- ------------------------------------- | time/ | | | fps | 323 | | iterations | 5500 | | time_elapsed | 84 | | total_timesteps | 27500 | | train/ | | | entropy_loss | -0.0149 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 5499 | | policy_loss | -1.13e-06 | | value_loss | 3.87e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 324 | | iterations | 5600 | | time_elapsed | 86 | | total_timesteps | 28000 | | train/ | | | entropy_loss | -0.0142 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 5599 | | policy_loss | 8.31e-07 | | value_loss | 2.3e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 323 | | iterations | 5700 | | time_elapsed | 87 | | total_timesteps | 28500 | | train/ | | | entropy_loss | -0.0139 | | explained_variance | -10.4 | | learning_rate | 0.0007 | | n_updates | 5699 | | policy_loss | -3.13e-06 | | value_loss | 4.23e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 323 | | iterations | 5800 | | time_elapsed | 89 | | total_timesteps | 29000 | | train/ | | | entropy_loss | -0.0133 | | explained_variance | -6.01e-05 | | learning_rate | 0.0007 | | n_updates | 5799 | | policy_loss | 1.89e-07 | | value_loss | 1.56e-08 | ------------------------------------- ------------------------------------ | time/ | | | fps | 323 | | iterations | 5900 | | time_elapsed | 91 | | total_timesteps | 29500 | | train/ | | | entropy_loss | -0.013 | | explained_variance | -6.75 | | learning_rate | 0.0007 | | n_updates | 5899 | | policy_loss | 1.93e-06 | | value_loss | 2.21e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 319 | | iterations | 6000 | | time_elapsed | 93 | | total_timesteps | 30000 | | train/ | | | entropy_loss | -0.0156 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 5999 | | policy_loss | 1.19e-06 | | value_loss | 3.89e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 316 | | iterations | 6100 | | time_elapsed | 96 | | total_timesteps | 30500 | | train/ | | | entropy_loss | -0.0217 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 6099 | | policy_loss | -8.86e-05 | | value_loss | 3.24e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 313 | | iterations | 6200 | | time_elapsed | 98 | | total_timesteps | 31000 | | train/ | | | entropy_loss | -0.0288 | | explained_variance | -18.4 | | learning_rate | 0.0007 | | n_updates | 6199 | | policy_loss | 1.51e-06 | | value_loss | 2.35e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 310 | | iterations | 6300 | | time_elapsed | 101 | | total_timesteps | 31500 | | train/ | | | entropy_loss | -0.0151 | | explained_variance | -0.371 | | learning_rate | 0.0007 | | n_updates | 6299 | | policy_loss | 3.38e-07 | | value_loss | 6.04e-08 | ------------------------------------ ------------------------------------ | time/ | | | fps | 307 | | iterations | 6400 | | time_elapsed | 104 | | total_timesteps | 32000 | | train/ | | | entropy_loss | -0.0108 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 6399 | | policy_loss | 1.74e-06 | | value_loss | 1.88e-06 | ------------------------------------ ------------------------------------- | time/ | | | fps | 303 | | iterations | 6500 | | time_elapsed | 107 | | total_timesteps | 32500 | | train/ | | | entropy_loss | -0.008 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 6499 | | policy_loss | 4.8e-07 | | value_loss | 4.41e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 300 | | iterations | 6600 | | time_elapsed | 109 | | total_timesteps | 33000 | | train/ | | | entropy_loss | -0.0077 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 6599 | | policy_loss | 1.41e-07 | | value_loss | 1.94e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 296 | | iterations | 6700 | | time_elapsed | 112 | | total_timesteps | 33500 | | train/ | | | entropy_loss | -0.00764 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 6699 | | policy_loss | 4.16e-07 | | value_loss | 2.53e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 292 | | iterations | 6800 | | time_elapsed | 116 | | total_timesteps | 34000 | | train/ | | | entropy_loss | -0.00738 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 6799 | | policy_loss | 1.8e-07 | | value_loss | 5.08e-08 | ------------------------------------- ------------------------------------- | time/ | | | fps | 291 | | iterations | 6900 | | time_elapsed | 118 | | total_timesteps | 34500 | | train/ | | | entropy_loss | -0.00749 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 6899 | | policy_loss | -6.73e-07 | | value_loss | 7.23e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 290 | | iterations | 7000 | | time_elapsed | 120 | | total_timesteps | 35000 | | train/ | | | entropy_loss | -0.00749 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 6999 | | policy_loss | -4.07e-07 | | value_loss | 3.65e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 289 | | iterations | 7100 | | time_elapsed | 122 | | total_timesteps | 35500 | | train/ | | | entropy_loss | -0.00455 | | explained_variance | -8.14 | | learning_rate | 0.0007 | | n_updates | 7099 | | policy_loss | 2.3e-07 | | value_loss | 4.69e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 289 | | iterations | 7200 | | time_elapsed | 124 | | total_timesteps | 36000 | | train/ | | | entropy_loss | -0.00469 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7199 | | policy_loss | -6.61e-08 | | value_loss | 1.93e-08 | ------------------------------------- ------------------------------------- | time/ | | | fps | 288 | | iterations | 7300 | | time_elapsed | 126 | | total_timesteps | 36500 | | train/ | | | entropy_loss | -0.00533 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7299 | | policy_loss | -8.97e-07 | | value_loss | 2.61e-06 | ------------------------------------- ------------------------------------- | time/ | | | fps | 288 | | iterations | 7400 | | time_elapsed | 128 | | total_timesteps | 37000 | | train/ | | | entropy_loss | -0.00551 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7399 | | policy_loss | -4.53e-08 | | value_loss | 1.7e-07 | ------------------------------------- ------------------------------------- | time/ | | | fps | 288 | | iterations | 7500 | | time_elapsed | 130 | | total_timesteps | 37500 | | train/ | | | entropy_loss | -0.0055 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7499 | | policy_loss | -1.91e-07 | | value_loss | 1.83e-07 | ------------------------------------- ------------------------------------ | time/ | | | fps | 287 | | iterations | 7600 | | time_elapsed | 132 | | total_timesteps | 38000 | | train/ | | | entropy_loss | -0.00544 | | explained_variance | -0.00785 | | learning_rate | 0.0007 | | n_updates | 7599 | | policy_loss | 6.53e-07 | | value_loss | 1.41e-06 | ------------------------------------ ------------------------------------ | time/ | | | fps | 287 | | iterations | 7700 | | time_elapsed | 133 | | total_timesteps | 38500 | | train/ | | | entropy_loss | -0.00537 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7699 | | policy_loss | 3.4e-07 | | value_loss | 4.42e-07 | ------------------------------------ ------------------------------------- | time/ | | | fps | 287 | | iterations | 7800 | | time_elapsed | 135 | | total_timesteps | 39000 | | train/ | | | entropy_loss | -0.00567 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7799 | | policy_loss | -6.66e-09 | | value_loss | 4.08e-08 | ------------------------------------- ------------------------------------ | time/ | | | fps | 286 | | iterations | 7900 | | time_elapsed | 137 | | total_timesteps | 39500 | | train/ | | | entropy_loss | -0.00558 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 7899 | | policy_loss | -3e-07 | | value_loss | 4.75e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 284 | | iterations | 8000 | | time_elapsed | 140 | | total_timesteps | 40000 | | train/ | | | entropy_loss | -0.00341 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 7999 | | policy_loss | -5.68e-07 | | value_loss | 3.09e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 283 | | iterations | 8100 | | time_elapsed | 142 | | total_timesteps | 40500 | | train/ | | | entropy_loss | -0.00343 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8099 | | policy_loss | 1.61e-07 | | value_loss | 2.61e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 283 | | iterations | 8200 | | time_elapsed | 144 | | total_timesteps | 41000 | | train/ | | | entropy_loss | -0.00334 | | explained_variance | 5.96e-08 | | learning_rate | 0.0007 | | n_updates | 8199 | | policy_loss | -1.11e-07 | | value_loss | 1.81e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 282 | | iterations | 8300 | | time_elapsed | 146 | | total_timesteps | 41500 | | train/ | | | entropy_loss | -0.00306 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8299 | | policy_loss | 1.19e-07 | | value_loss | 2.34e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 281 | | iterations | 8400 | | time_elapsed | 149 | | total_timesteps | 42000 | | train/ | | | entropy_loss | -0.00274 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8399 | | policy_loss | 1.79e-07 | | value_loss | 5.11e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 280 | | iterations | 8500 | | time_elapsed | 151 | | total_timesteps | 42500 | | train/ | | | entropy_loss | -0.00323 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8499 | | policy_loss | -2.51e-07 | | value_loss | 9.46e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 279 | | iterations | 8600 | | time_elapsed | 153 | | total_timesteps | 43000 | | train/ | | | entropy_loss | -0.0034 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8599 | | policy_loss | 3.08e-07 | | value_loss | 4e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 279 | | iterations | 8700 | | time_elapsed | 155 | | total_timesteps | 43500 | | train/ | | | entropy_loss | -0.00381 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 8699 | | policy_loss | 1.01e-07 | | value_loss | 9.43e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 279 | | iterations | 8800 | | time_elapsed | 157 | | total_timesteps | 44000 | | train/ | | | entropy_loss | -0.00322 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8799 | | policy_loss | -4.56e-07 | | value_loss | 2.21e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 278 | | iterations | 8900 | | time_elapsed | 159 | | total_timesteps | 44500 | | train/ | | | entropy_loss | -0.00289 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8899 | | policy_loss | -1.57e-07 | | value_loss | 3.32e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 277 | | iterations | 9000 | | time_elapsed | 162 | | total_timesteps | 45000 | | train/ | | | entropy_loss | -0.00307 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 8999 | | policy_loss | 9.29e-08 | | value_loss | 1.2e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 277 | | iterations | 9100 | | time_elapsed | 164 | | total_timesteps | 45500 | | train/ | | | entropy_loss | -0.00318 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 9099 | | policy_loss | 3.38e-07 | | value_loss | 1.18e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 277 | | iterations | 9200 | | time_elapsed | 165 | | total_timesteps | 46000 | | train/ | | | entropy_loss | -0.00259 | | explained_variance | -45.4 | | learning_rate | 0.0007 | | n_updates | 9199 | | policy_loss | 1.23e-08 | | value_loss | 2.24e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 277 | | iterations | 9300 | | time_elapsed | 167 | | total_timesteps | 46500 | | train/ | | | entropy_loss | -0.00279 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 9299 | | policy_loss | -1.45e-07 | | value_loss | 3.89e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 276 | | iterations | 9400 | | time_elapsed | 169 | | total_timesteps | 47000 | | train/ | | | entropy_loss | -0.00235 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 9399 | | policy_loss | -3.56e-07 | | value_loss | 2.23e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 276 | | iterations | 9500 | | time_elapsed | 171 | | total_timesteps | 47500 | | train/ | | | entropy_loss | -0.00241 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 9499 | | policy_loss | -2.54e-08 | | value_loss | 3.33e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 276 | | iterations | 9600 | | time_elapsed | 173 | | total_timesteps | 48000 | | train/ | | | entropy_loss | -0.00242 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 9599 | | policy_loss | 5.59e-08 | | value_loss | 4.24e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 275 | | iterations | 9700 | | time_elapsed | 175 | | total_timesteps | 48500 | | train/ | | | entropy_loss | -0.00674 | | explained_variance | -4.03 | | learning_rate | 0.0007 | | n_updates | 9699 | | policy_loss | -3.96e-07 | | value_loss | 1.78e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 275 | | iterations | 9800 | | time_elapsed | 178 | | total_timesteps | 49000 | | train/ | | | entropy_loss | -0.00224 | | explained_variance | 1.19e-05 | | learning_rate | 0.0007 | | n_updates | 9799 | | policy_loss | 2.55e-07 | | value_loss | 1.71e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 274 | | iterations | 9900 | | time_elapsed | 180 | | total_timesteps | 49500 | | train/ | | | entropy_loss | -0.00223 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 9899 | | policy_loss | 1.56e-07 | | value_loss | 8.44e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 273 | | iterations | 10000 | | time_elapsed | 182 | | total_timesteps | 50000 | | train/ | | | entropy_loss | -0.00223 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 9999 | | policy_loss | 2.31e-07 | | value_loss | 1.16e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10100 | | time_elapsed | 185 | | total_timesteps | 50500 | | train/ | | | entropy_loss | -0.00201 | | explained_variance | 9.94e-05 | | learning_rate | 0.0007 | | n_updates | 10099 | | policy_loss | 6.55e-08 | | value_loss | 1.2e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10200 | | time_elapsed | 187 | | total_timesteps | 51000 | | train/ | | | entropy_loss | -0.00163 | | explained_variance | -0.372 | | learning_rate | 0.0007 | | n_updates | 10199 | | policy_loss | 3.4e-07 | | value_loss | 5.94e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10300 | | time_elapsed | 189 | | total_timesteps | 51500 | | train/ | | | entropy_loss | -0.00199 | | explained_variance | 3.44e-05 | | learning_rate | 0.0007 | | n_updates | 10299 | | policy_loss | 3.58e-08 | | value_loss | 5.18e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10400 | | time_elapsed | 191 | | total_timesteps | 52000 | | train/ | | | entropy_loss | -0.00196 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 10399 | | policy_loss | 2.64e-07 | | value_loss | 2.31e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10500 | | time_elapsed | 192 | | total_timesteps | 52500 | | train/ | | | entropy_loss | -0.00241 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 10499 | | policy_loss | 6.48e-07 | | value_loss | 7.47e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10600 | | time_elapsed | 194 | | total_timesteps | 53000 | | train/ | | | entropy_loss | -0.00177 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 10599 | | policy_loss | -5.67e-08 | | value_loss | 1.37e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10700 | | time_elapsed | 196 | | total_timesteps | 53500 | | train/ | | | entropy_loss | -0.00176 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 10699 | | policy_loss | 7.04e-08 | | value_loss | 1.78e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10800 | | time_elapsed | 198 | | total_timesteps | 54000 | | train/ | | | entropy_loss | -0.00177 | | explained_variance | -2.07e-05 | | learning_rate | 0.0007 | | n_updates | 10799 | | policy_loss | 1.36e-08 | | value_loss | 5.07e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 272 | | iterations | 10900 | | time_elapsed | 200 | | total_timesteps | 54500 | | train/ | | | entropy_loss | -0.00199 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 10899 | | policy_loss | -2.52e-07 | | value_loss | 1.82e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 271 | | iterations | 11000 | | time_elapsed | 202 | | total_timesteps | 55000 | | train/ | | | entropy_loss | -0.00199 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 10999 | | policy_loss | -8.72e-08 | | value_loss | 4.42e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 271 | | iterations | 11100 | | time_elapsed | 204 | | total_timesteps | 55500 | | train/ | | | entropy_loss | -0.00184 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11099 | | policy_loss | 1.46e-08 | | value_loss | 3.99e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 271 | | iterations | 11200 | | time_elapsed | 206 | | total_timesteps | 56000 | | train/ | | | entropy_loss | -0.00179 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11199 | | policy_loss | 1.79e-07 | | value_loss | 1.24e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 270 | | iterations | 11300 | | time_elapsed | 208 | | total_timesteps | 56500 | | train/ | | | entropy_loss | -0.00179 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 11299 | | policy_loss | -1.3e-07 | | value_loss | 5.51e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 270 | | iterations | 11400 | | time_elapsed | 210 | | total_timesteps | 57000 | | train/ | | | entropy_loss | -0.00192 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 11399 | | policy_loss | 4.14e-08 | | value_loss | 2.51e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 270 | | iterations | 11500 | | time_elapsed | 212 | | total_timesteps | 57500 | | train/ | | | entropy_loss | -0.00185 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11499 | | policy_loss | 1.43e-07 | | value_loss | 6.49e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 11600 | | time_elapsed | 214 | | total_timesteps | 58000 | | train/ | | | entropy_loss | -0.00528 | | explained_variance | -0.607 | | learning_rate | 0.0007 | | n_updates | 11599 | | policy_loss | 4.24e-07 | | value_loss | 1.18e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 11700 | | time_elapsed | 216 | | total_timesteps | 58500 | | train/ | | | entropy_loss | -0.00188 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 11699 | | policy_loss | -2.21e-07 | | value_loss | 1.57e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 11800 | | time_elapsed | 218 | | total_timesteps | 59000 | | train/ | | | entropy_loss | -0.00189 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11799 | | policy_loss | -5.29e-08 | | value_loss | 1.73e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 11900 | | time_elapsed | 220 | | total_timesteps | 59500 | | train/ | | | entropy_loss | -0.00189 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11899 | | policy_loss | 7.49e-09 | | value_loss | 5.51e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12000 | | time_elapsed | 222 | | total_timesteps | 60000 | | train/ | | | entropy_loss | -0.00193 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 11999 | | policy_loss | 2.09e-07 | | value_loss | 1.2e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12100 | | time_elapsed | 224 | | total_timesteps | 60500 | | train/ | | | entropy_loss | -0.00155 | | explained_variance | -0.000509 | | learning_rate | 0.0007 | | n_updates | 12099 | | policy_loss | 1.02e-08 | | value_loss | 5.74e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12200 | | time_elapsed | 226 | | total_timesteps | 61000 | | train/ | | | entropy_loss | -0.0015 | | explained_variance | 0.0013 | | learning_rate | 0.0007 | | n_updates | 12199 | | policy_loss | -3.54e-08 | | value_loss | 6.87e-08 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12300 | | time_elapsed | 228 | | total_timesteps | 61500 | | train/ | | | entropy_loss | -0.00197 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 12299 | | policy_loss | 4.29e-07 | | value_loss | 5.15e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12400 | | time_elapsed | 229 | | total_timesteps | 62000 | | train/ | | | entropy_loss | -0.00196 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12399 | | policy_loss | 1.13e-07 | | value_loss | 4.86e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12500 | | time_elapsed | 231 | | total_timesteps | 62500 | | train/ | | | entropy_loss | -0.00196 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12499 | | policy_loss | -3.7e-08 | | value_loss | 7.61e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12600 | | time_elapsed | 233 | | total_timesteps | 63000 | | train/ | | | entropy_loss | -0.00308 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12599 | | policy_loss | 2.54e-07 | | value_loss | 6.8e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12700 | | time_elapsed | 235 | | total_timesteps | 63500 | | train/ | | | entropy_loss | -0.0029 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12699 | | policy_loss | -2.17e-08 | | value_loss | 2.25e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 269 | | iterations | 12800 | | time_elapsed | 237 | | total_timesteps | 64000 | | train/ | | | entropy_loss | -0.00306 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 12799 | | policy_loss | 3.1e-07 | | value_loss | 1.18e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 12900 | | time_elapsed | 239 | | total_timesteps | 64500 | | train/ | | | entropy_loss | -0.00327 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12899 | | policy_loss | -9.89e-08 | | value_loss | 3.26e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13000 | | time_elapsed | 241 | | total_timesteps | 65000 | | train/ | | | entropy_loss | -0.00329 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 12999 | | policy_loss | 2.77e-08 | | value_loss | 9.15e-08 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13100 | | time_elapsed | 243 | | total_timesteps | 65500 | | train/ | | | entropy_loss | -0.00334 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 13099 | | policy_loss | -2.29e-07 | | value_loss | 6.52e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13200 | | time_elapsed | 246 | | total_timesteps | 66000 | | train/ | | | entropy_loss | -0.00334 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 13199 | | policy_loss | 1.73e-07 | | value_loss | 2.65e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13300 | | time_elapsed | 248 | | total_timesteps | 66500 | | train/ | | | entropy_loss | -0.00334 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 13299 | | policy_loss | 5.51e-07 | | value_loss | 3.01e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13400 | | time_elapsed | 249 | | total_timesteps | 67000 | | train/ | | | entropy_loss | -0.00334 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 13399 | | policy_loss | 1.97e-07 | | value_loss | 4.24e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13500 | | time_elapsed | 251 | | total_timesteps | 67500 | | train/ | | | entropy_loss | -0.039 | | explained_variance | -26.7 | | learning_rate | 0.0007 | | n_updates | 13499 | | policy_loss | -1.84e-06 | | value_loss | 8.37e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 268 | | iterations | 13600 | | time_elapsed | 253 | | total_timesteps | 68000 | | train/ | | | entropy_loss | -0.00405 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 13599 | | policy_loss | -2.37e-07 | | value_loss | 3.07e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 267 | | iterations | 13700 | | time_elapsed | 255 | | total_timesteps | 68500 | | train/ | | | entropy_loss | -0.0041 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 13699 | | policy_loss | 1.88e-07 | | value_loss | 2.35e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 267 | | iterations | 13800 | | time_elapsed | 258 | | total_timesteps | 69000 | | train/ | | | entropy_loss | -0.0039 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 13799 | | policy_loss | 3e-08 | | value_loss | 5.83e-08 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 266 | | iterations | 13900 | | time_elapsed | 260 | | total_timesteps | 69500 | | train/ | | | entropy_loss | -0.0039 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 13899 | | policy_loss | 5.74e-08 | | value_loss | 3.5e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 266 | | iterations | 14000 | | time_elapsed | 262 | | total_timesteps | 70000 | | train/ | | | entropy_loss | -0.339 | | explained_variance | -2.68 | | learning_rate | 0.0007 | | n_updates | 13999 | | policy_loss | -0.00017 | | value_loss | 2.91e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 266 | | iterations | 14100 | | time_elapsed | 264 | | total_timesteps | 70500 | | train/ | | | entropy_loss | -0.00242 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 14099 | | policy_loss | -8.7e-08 | | value_loss | 1.49e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 14200 | | time_elapsed | 266 | | total_timesteps | 71000 | | train/ | | | entropy_loss | -0.00264 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14199 | | policy_loss | -1.67e-07 | | value_loss | 4.86e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 14300 | | time_elapsed | 269 | | total_timesteps | 71500 | | train/ | | | entropy_loss | -0.00252 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14299 | | policy_loss | -2.56e-07 | | value_loss | 1.09e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 14400 | | time_elapsed | 271 | | total_timesteps | 72000 | | train/ | | | entropy_loss | -0.00227 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14399 | | policy_loss | -2.36e-07 | | value_loss | 1.11e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 14500 | | time_elapsed | 273 | | total_timesteps | 72500 | | train/ | | | entropy_loss | -0.00247 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14499 | | policy_loss | -1.91e-07 | | value_loss | 5.9e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 14600 | | time_elapsed | 275 | | total_timesteps | 73000 | | train/ | | | entropy_loss | -0.00272 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 14599 | | policy_loss | 1.74e-07 | | value_loss | 4.13e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 14700 | | time_elapsed | 277 | | total_timesteps | 73500 | | train/ | | | entropy_loss | -0.00286 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 14699 | | policy_loss | -1.16e-07 | | value_loss | 1.55e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 14800 | | time_elapsed | 279 | | total_timesteps | 74000 | | train/ | | | entropy_loss | -0.00286 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14799 | | policy_loss | 1.03e-07 | | value_loss | 1.67e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 14900 | | time_elapsed | 281 | | total_timesteps | 74500 | | train/ | | | entropy_loss | -0.00252 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14899 | | policy_loss | 1.68e-07 | | value_loss | 5e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15000 | | time_elapsed | 282 | | total_timesteps | 75000 | | train/ | | | entropy_loss | -0.00282 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 14999 | | policy_loss | -1.24e-07 | | value_loss | 1.88e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 15100 | | time_elapsed | 284 | | total_timesteps | 75500 | | train/ | | | entropy_loss | -0.00245 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 15099 | | policy_loss | 2.49e-08 | | value_loss | 4.17e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15200 | | time_elapsed | 286 | | total_timesteps | 76000 | | train/ | | | entropy_loss | -0.0025 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 15199 | | policy_loss | -7.05e-08 | | value_loss | 1.74e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15300 | | time_elapsed | 288 | | total_timesteps | 76500 | | train/ | | | entropy_loss | -0.00231 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 15299 | | policy_loss | -6.79e-08 | | value_loss | 1.08e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15400 | | time_elapsed | 290 | | total_timesteps | 77000 | | train/ | | | entropy_loss | -0.00212 | | explained_variance | -0.00223 | | learning_rate | 0.0007 | | n_updates | 15399 | | policy_loss | 6.95e-08 | | value_loss | 1.41e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15500 | | time_elapsed | 292 | | total_timesteps | 77500 | | train/ | | | entropy_loss | -0.00218 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 15499 | | policy_loss | -2.11e-08 | | value_loss | 1.85e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 265 | | iterations | 15600 | | time_elapsed | 294 | | total_timesteps | 78000 | | train/ | | | entropy_loss | -0.00224 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 15599 | | policy_loss | -4.52e-08 | | value_loss | 5.43e-08 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 15700 | | time_elapsed | 296 | | total_timesteps | 78500 | | train/ | | | entropy_loss | -0.00224 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 15699 | | policy_loss | 2.38e-08 | | value_loss | 1.14e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.548 | | time/ | | | fps | 264 | | iterations | 15800 | | time_elapsed | 298 | | total_timesteps | 79000 | | train/ | | | entropy_loss | -0.00225 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 15799 | | policy_loss | -1.93e-08 | | value_loss | 2.79e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 264 | | iterations | 15900 | | time_elapsed | 300 | | total_timesteps | 79500 | | train/ | | | entropy_loss | -0.00476 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 15899 | | policy_loss | 4.02e-07 | | value_loss | 7.96e-07 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 263 | | iterations | 16000 | | time_elapsed | 303 | | total_timesteps | 80000 | | train/ | | | entropy_loss | -0.00489 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 15999 | | policy_loss | 8.73e-07 | | value_loss | 3.07e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 262 | | iterations | 16100 | | time_elapsed | 306 | | total_timesteps | 80500 | | train/ | | | entropy_loss | -0.00329 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16099 | | policy_loss | 5.7e-08 | | value_loss | 2.39e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 262 | | iterations | 16200 | | time_elapsed | 308 | | total_timesteps | 81000 | | train/ | | | entropy_loss | -0.003 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 16199 | | policy_loss | -8.02e-08 | | value_loss | 1.06e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 261 | | iterations | 16300 | | time_elapsed | 311 | | total_timesteps | 81500 | | train/ | | | entropy_loss | -0.00325 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16299 | | policy_loss | -3.32e-07 | | value_loss | 1.15e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 261 | | iterations | 16400 | | time_elapsed | 313 | | total_timesteps | 82000 | | train/ | | | entropy_loss | -0.00312 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16399 | | policy_loss | -3.28e-07 | | value_loss | 1.56e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 16500 | | time_elapsed | 316 | | total_timesteps | 82500 | | train/ | | | entropy_loss | -0.00269 | | explained_variance | 0.000353 | | learning_rate | 0.0007 | | n_updates | 16499 | | policy_loss | -1.24e-08 | | value_loss | 5.1e-09 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 16600 | | time_elapsed | 318 | | total_timesteps | 83000 | | train/ | | | entropy_loss | -0.00254 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16599 | | policy_loss | -9.92e-08 | | value_loss | 1.73e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 16700 | | time_elapsed | 320 | | total_timesteps | 83500 | | train/ | | | entropy_loss | -0.0025 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 16699 | | policy_loss | 3.89e-07 | | value_loss | 2.8e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 16800 | | time_elapsed | 322 | | total_timesteps | 84000 | | train/ | | | entropy_loss | -0.0025 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16799 | | policy_loss | 2.1e-07 | | value_loss | 9.55e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 16900 | | time_elapsed | 324 | | total_timesteps | 84500 | | train/ | | | entropy_loss | -0.00249 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 16899 | | policy_loss | 1.48e-07 | | value_loss | 3.85e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 17000 | | time_elapsed | 326 | | total_timesteps | 85000 | | train/ | | | entropy_loss | -0.00503 | | explained_variance | -1.85 | | learning_rate | 0.0007 | | n_updates | 16999 | | policy_loss | -3.69e-07 | | value_loss | 8.96e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 260 | | iterations | 17100 | | time_elapsed | 328 | | total_timesteps | 85500 | | train/ | | | entropy_loss | -0.0023 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17099 | | policy_loss | -5.25e-09 | | value_loss | 1.53e-08 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17200 | | time_elapsed | 330 | | total_timesteps | 86000 | | train/ | | | entropy_loss | -0.00249 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17199 | | policy_loss | 2.65e-07 | | value_loss | 1.3e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17300 | | time_elapsed | 332 | | total_timesteps | 86500 | | train/ | | | entropy_loss | -0.00247 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17299 | | policy_loss | 9.03e-08 | | value_loss | 2.69e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17400 | | time_elapsed | 334 | | total_timesteps | 87000 | | train/ | | | entropy_loss | -0.00132 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17399 | | policy_loss | 8.05e-08 | | value_loss | 4.99e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17500 | | time_elapsed | 337 | | total_timesteps | 87500 | | train/ | | | entropy_loss | -0.00468 | | explained_variance | -5.09 | | learning_rate | 0.0007 | | n_updates | 17499 | | policy_loss | -1.33e-07 | | value_loss | 5.81e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17600 | | time_elapsed | 339 | | total_timesteps | 88000 | | train/ | | | entropy_loss | -0.00132 | | explained_variance | -0.00905 | | learning_rate | 0.0007 | | n_updates | 17599 | | policy_loss | 2.64e-08 | | value_loss | 5.46e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17700 | | time_elapsed | 341 | | total_timesteps | 88500 | | train/ | | | entropy_loss | -0.00153 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17699 | | policy_loss | 2.11e-07 | | value_loss | 2.43e-06 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17800 | | time_elapsed | 342 | | total_timesteps | 89000 | | train/ | | | entropy_loss | -0.00149 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17799 | | policy_loss | -4.03e-07 | | value_loss | 8.55e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 17900 | | time_elapsed | 344 | | total_timesteps | 89500 | | train/ | | | entropy_loss | -0.00149 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 17899 | | policy_loss | 5.53e-08 | | value_loss | 3.97e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18000 | | time_elapsed | 346 | | total_timesteps | 90000 | | train/ | | | entropy_loss | -0.00146 | | explained_variance | 0.000154 | | learning_rate | 0.0007 | | n_updates | 17999 | | policy_loss | 5.49e-08 | | value_loss | 1.86e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18100 | | time_elapsed | 348 | | total_timesteps | 90500 | | train/ | | | entropy_loss | -0.00143 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 18099 | | policy_loss | 1.66e-08 | | value_loss | 2.37e-08 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18200 | | time_elapsed | 350 | | total_timesteps | 91000 | | train/ | | | entropy_loss | -0.00167 | | explained_variance | 5.96e-08 | | learning_rate | 0.0007 | | n_updates | 18199 | | policy_loss | 9.68e-08 | | value_loss | 5.07e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18300 | | time_elapsed | 352 | | total_timesteps | 91500 | | train/ | | | entropy_loss | -0.00174 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 18299 | | policy_loss | 7.17e-07 | | value_loss | 2.34e-05 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18400 | | time_elapsed | 353 | | total_timesteps | 92000 | | train/ | | | entropy_loss | -0.00109 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 18399 | | policy_loss | 5.21e-08 | | value_loss | 3.2e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18500 | | time_elapsed | 356 | | total_timesteps | 92500 | | train/ | | | entropy_loss | -0.00121 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 18499 | | policy_loss | -2.66e-08 | | value_loss | 1.62e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18600 | | time_elapsed | 358 | | total_timesteps | 93000 | | train/ | | | entropy_loss | -0.00119 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 18599 | | policy_loss | -2.02e-08 | | value_loss | 4.06e-08 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18700 | | time_elapsed | 360 | | total_timesteps | 93500 | | train/ | | | entropy_loss | -0.00121 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 18699 | | policy_loss | 1.87e-07 | | value_loss | 3.02e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 18800 | | time_elapsed | 362 | | total_timesteps | 94000 | | train/ | | | entropy_loss | -0.00118 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 18799 | | policy_loss | 6.23e-08 | | value_loss | 4.65e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 18900 | | time_elapsed | 365 | | total_timesteps | 94500 | | train/ | | | entropy_loss | -0.00108 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 18899 | | policy_loss | 2.17e-08 | | value_loss | 5.12e-08 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19000 | | time_elapsed | 367 | | total_timesteps | 95000 | | train/ | | | entropy_loss | -0.0012 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 18999 | | policy_loss | -2.02e-07 | | value_loss | 4.04e-06 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19100 | | time_elapsed | 369 | | total_timesteps | 95500 | | train/ | | | entropy_loss | -0.00114 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19099 | | policy_loss | 5.66e-08 | | value_loss | 3.24e-07 | ------------------------------------ ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19200 | | time_elapsed | 371 | | total_timesteps | 96000 | | train/ | | | entropy_loss | -0.00104 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19199 | | policy_loss | -4.69e-08 | | value_loss | 2.49e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19300 | | time_elapsed | 373 | | total_timesteps | 96500 | | train/ | | | entropy_loss | -0.00105 | | explained_variance | -1.19e-07 | | learning_rate | 0.0007 | | n_updates | 19299 | | policy_loss | 1.12e-07 | | value_loss | 2.48e-06 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19400 | | time_elapsed | 375 | | total_timesteps | 97000 | | train/ | | | entropy_loss | -0.0239 | | explained_variance | 0.186 | | learning_rate | 0.0007 | | n_updates | 19399 | | policy_loss | -2.59e-06 | | value_loss | 2.3e-07 | ------------------------------------- ------------------------------------- | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19500 | | time_elapsed | 376 | | total_timesteps | 97500 | | train/ | | | entropy_loss | -0.00119 | | explained_variance | 1.19e-07 | | learning_rate | 0.0007 | | n_updates | 19499 | | policy_loss | -5.15e-09 | | value_loss | 3.86e-08 | ------------------------------------- ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19600 | | time_elapsed | 378 | | total_timesteps | 98000 | | train/ | | | entropy_loss | -0.00122 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19599 | | policy_loss | 4.79e-08 | | value_loss | 2.26e-07 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19700 | | time_elapsed | 380 | | total_timesteps | 98500 | | train/ | | | entropy_loss | -0.00124 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19699 | | policy_loss | 2.39e-07 | | value_loss | 4.65e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 19800 | | time_elapsed | 382 | | total_timesteps | 99000 | | train/ | | | entropy_loss | -0.00129 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19799 | | policy_loss | -1.7e-07 | | value_loss | 2.56e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 259 | | iterations | 19900 | | time_elapsed | 384 | | total_timesteps | 99500 | | train/ | | | entropy_loss | -0.00122 | | explained_variance | -4.45 | | learning_rate | 0.0007 | | n_updates | 19899 | | policy_loss | 4.92e-08 | | value_loss | 1.89e-06 | ------------------------------------ ------------------------------------ | rollout/ | | | ep_len_mean | 3.97e+04 | | ep_rew_mean | -0.259 | | time/ | | | fps | 258 | | iterations | 20000 | | time_elapsed | 386 | | total_timesteps | 100000 | | train/ | | | entropy_loss | -0.00103 | | explained_variance | 0 | | learning_rate | 0.0007 | | n_updates | 19999 | | policy_loss | 4.79e-08 | | value_loss | 2.66e-07 | ------------------------------------ Saved A2C model to: G:\My Drive\Bots DRL\DRL\DRL-MT5-Lab\notebooks\models\a2c_EURUSD_M15.zip
In [4]:
# === Train DQN (same env/data as PPO) ===
import os, json
from pathlib import Path
from stable_baselines3 import DQN
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3.common.monitor import Monitor
def make_env():
return Monitor(TradingEnv(df_train, feature_cols))
env_dqn = DummyVecEnv([make_env])
dqn = DQN("MlpPolicy", env_dqn,
verbose=1,
learning_rate=1e-3,
buffer_size=100_000,
learning_starts=1_000,
batch_size=64,
tau=1.0,
gamma=0.99,
train_freq=4,
target_update_interval=1_000,
exploration_fraction=0.1,
exploration_final_eps=0.01)
total_timesteps = int(os.getenv("TOTAL_TIMESTEPS", "100000"))
dqn.learn(total_timesteps=total_timesteps)
Path("models").mkdir(exist_ok=True)
dqn_path = Path("models") / f"dqn_{SYMBOL}_{TIMEFRAME}.zip"
dqn.save(dqn_path.as_posix())
# (Write once is enough; safe to overwrite)
with open(Path("models") / "selected_features.json", "w", encoding="utf-8") as f:
json.dump(feature_cols, f, indent=2)
print("Saved DQN model to:", dqn_path.resolve())
Using cpu device Saved DQN model to: G:\My Drive\Bots DRL\DRL\DRL-MT5-Lab\notebooks\models\dqn_EURUSD_M15.zip