gavindiaz/Deep-Reinforcement-Learning-MT5-Bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
master
Deep-Reinforcement-Learning…/notebooks/2_Train.ipynb
T
Mohammad Aghdam 90d2259345 Initial commit
2025-10-05 07:52:39 +02:00

4332 lines
205 KiB
Python
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "markdown",
"id": "9b2bd831",
"metadata": {},
"source": [
"# 2) Train — Features & PPO (SB3)\n",
"\n",
"- Builds indicators via `features.add_indicators`.\n",
"- Trains a PPO, A2C and DQN agent on a minimal discrete env defined **in this notebook**.\n",
"- Saves model and the list of feature columns used."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "a7ff95b5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train: (39672, 17) Val: (9918, 17) Features: 17\n"
]
}
],
"source": [
"import sys, os\n",
"sys.path.insert(0, os.path.abspath('..')) # Path fix\n",
"\n",
"import os, json\n",
"from pathlib import Path\n",
"import numpy as np\n",
"import pandas as pd\n",
"from dotenv import load_dotenv\n",
"\n",
"from stable_baselines3 import PPO\n",
"from stable_baselines3.common.vec_env import DummyVecEnv\n",
"import gymnasium as gym\n",
"from gymnasium import spaces\n",
"\n",
"import features # your features.py\n",
"\n",
"load_dotenv()\n",
"SYMBOL = os.getenv(\"TRAINING_SYMBOL\", \"EURUSD\")\n",
"TIMEFRAME = os.getenv(\"TIMEFRAME\", \"M15\")\n",
"SPLIT_RATIO = float(os.getenv(\"SPLIT_RATIO\", \"0.8\"))\n",
"\n",
"DATA_CSV = Path(\"data\") / f\"ohlc_{SYMBOL}_{TIMEFRAME}.csv\"\n",
"assert DATA_CSV.exists(), f\"Missing dataset {DATA_CSV}. Run 1_Data.ipynb or provide CSV.\"\n",
"\n",
"df = pd.read_csv(DATA_CSV, parse_dates=[\"time\"], index_col=\"time\")\n",
"df_feat = features.add_indicators(df.copy())\n",
"\n",
"# --- ADD THIS LINE ---\n",
"df_feat.dropna(inplace=True)\n",
"# ---------------------\n",
"\n",
"# Choose numeric columns as features (exclude obvious targets if any)\n",
"candidates = df_feat.select_dtypes(include=[np.number]).columns.tolist()\n",
"# Keep OHLCV + indicators for now\n",
"feature_cols = candidates\n",
"\n",
"# Split\n",
"n_split = int(len(df_feat) * SPLIT_RATIO)\n",
"df_train = df_feat.iloc[:n_split].copy()\n",
"df_val = df_feat.iloc[n_split:].copy()\n",
"\n",
"print(\"Train:\", df_train.shape, \"Val:\", df_val.shape, \"Features:\", len(feature_cols))\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "70f0fde9",
"metadata": {},
"outputs": [],
"source": [
"\n",
"# Minimal discrete trading environment (buy/hold/sell).\n",
"class TradingEnv(gym.Env):\n",
" metadata = {\"render_modes\": []}\n",
" def __init__(self, df_feat, feature_cols, trade_cost=1e-4):\n",
" super().__init__()\n",
" self.df = df_feat\n",
" self.cols = feature_cols\n",
" self.trade_cost = float(trade_cost)\n",
" self.n = len(self.df)\n",
" self.idx = 0\n",
" self.position = 0 # -1, 0, +1\n",
" self.observation_space = spaces.Box(\n",
" low=-np.inf, high=np.inf, shape=(len(self.cols),), dtype=np.float32\n",
" )\n",
" self.action_space = spaces.Discrete(3) # 0=sell, 1=hold, 2=buy\n",
"\n",
" def _obs(self):\n",
" row = self.df.iloc[self.idx][self.cols].astype(float).values\n",
" return row.astype(np.float32)\n",
"\n",
" def reset(self, seed=None, options=None):\n",
" super().reset(seed=seed)\n",
" self.idx = 1 # need a previous bar for return\n",
" self.position = 0\n",
" obs = self._obs()\n",
" return obs, {}\n",
"\n",
" def step(self, action):\n",
" # map action -> position\n",
" pos_new = {-1:0, 0:0, 1:0, 2:1}[action] if action in (-1,0,1,2) else 0\n",
" pos_new = {-1: -1, 0: 0, 1: 1}.get({0:-1,1:0,2:1}[action], 0)\n",
"\n",
" # price return from t-1 -> t on close\n",
" prev = self.df[\"close\"].iloc[self.idx-1]\n",
" curr = self.df[\"close\"].iloc[self.idx]\n",
" ret = (curr - prev) / (prev + 1e-12)\n",
"\n",
" # reward is position * return minus cost if changed position\n",
" reward = pos_new * ret - (self.trade_cost if pos_new != self.position else 0.0)\n",
" self.position = pos_new\n",
"\n",
" # next\n",
" terminated = False\n",
" self.idx += 1\n",
" truncated = self.idx >= (self.n - 1)\n",
" obs = self._obs()\n",
" info = {\"position\": self.position, \"ret\": ret}\n",
" return obs, float(reward), terminated, truncated, info"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "2144f855",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cpu device\n",
"-----------------------------\n",
"| time/ | |\n",
"| fps | 710 |\n",
"| iterations | 1 |\n",
"| time_elapsed | 2 |\n",
"| total_timesteps | 2048 |\n",
"-----------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 587 |\n",
"| iterations | 2 |\n",
"| time_elapsed | 6 |\n",
"| total_timesteps | 4096 |\n",
"| train/ | |\n",
"| approx_kl | 0.009429634 |\n",
"| clip_fraction | 0.0859 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1.09 |\n",
"| explained_variance | -8.04 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0214 |\n",
"| n_updates | 10 |\n",
"| policy_gradient_loss | -0.00715 |\n",
"| value_loss | 0.00153 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 551 |\n",
"| iterations | 3 |\n",
"| time_elapsed | 11 |\n",
"| total_timesteps | 6144 |\n",
"| train/ | |\n",
"| approx_kl | 0.0042742263 |\n",
"| clip_fraction | 0.00928 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1.09 |\n",
"| explained_variance | -9.47 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00427 |\n",
"| n_updates | 20 |\n",
"| policy_gradient_loss | -0.00337 |\n",
"| value_loss | 0.000417 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 536 |\n",
"| iterations | 4 |\n",
"| time_elapsed | 15 |\n",
"| total_timesteps | 8192 |\n",
"| train/ | |\n",
"| approx_kl | 0.0044627683 |\n",
"| clip_fraction | 0.0359 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1.06 |\n",
"| explained_variance | -8.19 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00498 |\n",
"| n_updates | 30 |\n",
"| policy_gradient_loss | -0.00413 |\n",
"| value_loss | 0.000232 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 508 |\n",
"| iterations | 5 |\n",
"| time_elapsed | 20 |\n",
"| total_timesteps | 10240 |\n",
"| train/ | |\n",
"| approx_kl | 0.0061096027 |\n",
"| clip_fraction | 0.0265 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1.03 |\n",
"| explained_variance | -6.02 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0174 |\n",
"| n_updates | 40 |\n",
"| policy_gradient_loss | -0.01 |\n",
"| value_loss | 0.000184 |\n",
"------------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 503 |\n",
"| iterations | 6 |\n",
"| time_elapsed | 24 |\n",
"| total_timesteps | 12288 |\n",
"| train/ | |\n",
"| approx_kl | 0.008018628 |\n",
"| clip_fraction | 0.0893 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1.03 |\n",
"| explained_variance | -4.55 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.022 |\n",
"| n_updates | 50 |\n",
"| policy_gradient_loss | -0.00688 |\n",
"| value_loss | 0.000197 |\n",
"-----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 7 |\n",
"| time_elapsed | 28 |\n",
"| total_timesteps | 14336 |\n",
"| train/ | |\n",
"| approx_kl | 0.008238241 |\n",
"| clip_fraction | 0.0252 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -1 |\n",
"| explained_variance | -5 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0208 |\n",
"| n_updates | 60 |\n",
"| policy_gradient_loss | -0.00627 |\n",
"| value_loss | 7.93e-05 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 502 |\n",
"| iterations | 8 |\n",
"| time_elapsed | 32 |\n",
"| total_timesteps | 16384 |\n",
"| train/ | |\n",
"| approx_kl | 0.0063306787 |\n",
"| clip_fraction | 0.0549 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.952 |\n",
"| explained_variance | -6.96 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0044 |\n",
"| n_updates | 70 |\n",
"| policy_gradient_loss | -0.00798 |\n",
"| value_loss | 7.12e-05 |\n",
"------------------------------------------\n",
"----------------------------------------\n",
"| time/ | |\n",
"| fps | 502 |\n",
"| iterations | 9 |\n",
"| time_elapsed | 36 |\n",
"| total_timesteps | 18432 |\n",
"| train/ | |\n",
"| approx_kl | 0.00408049 |\n",
"| clip_fraction | 0.0551 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.932 |\n",
"| explained_variance | -2.84 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.017 |\n",
"| n_updates | 80 |\n",
"| policy_gradient_loss | -0.00532 |\n",
"| value_loss | 4.31e-05 |\n",
"----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 10 |\n",
"| time_elapsed | 40 |\n",
"| total_timesteps | 20480 |\n",
"| train/ | |\n",
"| approx_kl | 0.008295992 |\n",
"| clip_fraction | 0.0611 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.935 |\n",
"| explained_variance | -8.56 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0135 |\n",
"| n_updates | 90 |\n",
"| policy_gradient_loss | -0.00899 |\n",
"| value_loss | 5.76e-05 |\n",
"-----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 506 |\n",
"| iterations | 11 |\n",
"| time_elapsed | 44 |\n",
"| total_timesteps | 22528 |\n",
"| train/ | |\n",
"| approx_kl | 0.014076492 |\n",
"| clip_fraction | 0.0576 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.879 |\n",
"| explained_variance | -4.39 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0102 |\n",
"| n_updates | 100 |\n",
"| policy_gradient_loss | -0.00607 |\n",
"| value_loss | 2.66e-05 |\n",
"-----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 503 |\n",
"| iterations | 12 |\n",
"| time_elapsed | 48 |\n",
"| total_timesteps | 24576 |\n",
"| train/ | |\n",
"| approx_kl | 0.008959841 |\n",
"| clip_fraction | 0.068 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.912 |\n",
"| explained_variance | -7.66 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0189 |\n",
"| n_updates | 110 |\n",
"| policy_gradient_loss | -0.00744 |\n",
"| value_loss | 2.75e-05 |\n",
"-----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 500 |\n",
"| iterations | 13 |\n",
"| time_elapsed | 53 |\n",
"| total_timesteps | 26624 |\n",
"| train/ | |\n",
"| approx_kl | 0.008437004 |\n",
"| clip_fraction | 0.0575 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.903 |\n",
"| explained_variance | -4.35 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0319 |\n",
"| n_updates | 120 |\n",
"| policy_gradient_loss | -0.00935 |\n",
"| value_loss | 7.75e-05 |\n",
"-----------------------------------------\n",
"---------------------------------------\n",
"| time/ | |\n",
"| fps | 502 |\n",
"| iterations | 14 |\n",
"| time_elapsed | 57 |\n",
"| total_timesteps | 28672 |\n",
"| train/ | |\n",
"| approx_kl | 0.0126811 |\n",
"| clip_fraction | 0.119 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.816 |\n",
"| explained_variance | -5.55 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0323 |\n",
"| n_updates | 130 |\n",
"| policy_gradient_loss | -0.00794 |\n",
"| value_loss | 0.000615 |\n",
"---------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 503 |\n",
"| iterations | 15 |\n",
"| time_elapsed | 61 |\n",
"| total_timesteps | 30720 |\n",
"| train/ | |\n",
"| approx_kl | 0.0075738453 |\n",
"| clip_fraction | 0.0583 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.722 |\n",
"| explained_variance | -10.5 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0262 |\n",
"| n_updates | 140 |\n",
"| policy_gradient_loss | -0.00878 |\n",
"| value_loss | 1.59e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 16 |\n",
"| time_elapsed | 65 |\n",
"| total_timesteps | 32768 |\n",
"| train/ | |\n",
"| approx_kl | 0.0042516133 |\n",
"| clip_fraction | 0.0463 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.624 |\n",
"| explained_variance | -16.7 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0243 |\n",
"| n_updates | 150 |\n",
"| policy_gradient_loss | -0.00972 |\n",
"| value_loss | 1.29e-05 |\n",
"------------------------------------------\n",
"----------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 17 |\n",
"| time_elapsed | 68 |\n",
"| total_timesteps | 34816 |\n",
"| train/ | |\n",
"| approx_kl | 0.00597096 |\n",
"| clip_fraction | 0.0776 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.544 |\n",
"| explained_variance | -5.51 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00566 |\n",
"| n_updates | 160 |\n",
"| policy_gradient_loss | -0.00572 |\n",
"| value_loss | 1.4e-05 |\n",
"----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 18 |\n",
"| time_elapsed | 73 |\n",
"| total_timesteps | 36864 |\n",
"| train/ | |\n",
"| approx_kl | 0.005309558 |\n",
"| clip_fraction | 0.0558 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.525 |\n",
"| explained_variance | -9.82 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00756 |\n",
"| n_updates | 170 |\n",
"| policy_gradient_loss | -0.00672 |\n",
"| value_loss | 5.88e-05 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 19 |\n",
"| time_elapsed | 76 |\n",
"| total_timesteps | 38912 |\n",
"| train/ | |\n",
"| approx_kl | 0.0060142255 |\n",
"| clip_fraction | 0.0741 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.477 |\n",
"| explained_variance | -3.59 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0209 |\n",
"| n_updates | 180 |\n",
"| policy_gradient_loss | -0.00457 |\n",
"| value_loss | 1.85e-05 |\n",
"------------------------------------------\n",
"----------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 20 |\n",
"| time_elapsed | 81 |\n",
"| total_timesteps | 40960 |\n",
"| train/ | |\n",
"| approx_kl | 0.00463128 |\n",
"| clip_fraction | 0.0469 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.407 |\n",
"| explained_variance | -4.95 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0257 |\n",
"| n_updates | 190 |\n",
"| policy_gradient_loss | -0.00473 |\n",
"| value_loss | 0.00014 |\n",
"----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 504 |\n",
"| iterations | 21 |\n",
"| time_elapsed | 85 |\n",
"| total_timesteps | 43008 |\n",
"| train/ | |\n",
"| approx_kl | 0.0023931228 |\n",
"| clip_fraction | 0.0385 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.414 |\n",
"| explained_variance | -4.5 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0209 |\n",
"| n_updates | 200 |\n",
"| policy_gradient_loss | -0.005 |\n",
"| value_loss | 9.36e-06 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 22 |\n",
"| time_elapsed | 89 |\n",
"| total_timesteps | 45056 |\n",
"| train/ | |\n",
"| approx_kl | 0.0034446488 |\n",
"| clip_fraction | 0.0373 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.354 |\n",
"| explained_variance | -12.8 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0109 |\n",
"| n_updates | 210 |\n",
"| policy_gradient_loss | -0.0059 |\n",
"| value_loss | 7.07e-05 |\n",
"------------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 506 |\n",
"| iterations | 23 |\n",
"| time_elapsed | 93 |\n",
"| total_timesteps | 47104 |\n",
"| train/ | |\n",
"| approx_kl | 0.002063186 |\n",
"| clip_fraction | 0.0291 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.302 |\n",
"| explained_variance | -5.94 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.000529 |\n",
"| n_updates | 220 |\n",
"| policy_gradient_loss | -0.006 |\n",
"| value_loss | 2.82e-05 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 24 |\n",
"| time_elapsed | 97 |\n",
"| total_timesteps | 49152 |\n",
"| train/ | |\n",
"| approx_kl | 0.0032632346 |\n",
"| clip_fraction | 0.0346 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.364 |\n",
"| explained_variance | -5.95 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0142 |\n",
"| n_updates | 230 |\n",
"| policy_gradient_loss | -0.00837 |\n",
"| value_loss | 1.7e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 506 |\n",
"| iterations | 25 |\n",
"| time_elapsed | 101 |\n",
"| total_timesteps | 51200 |\n",
"| train/ | |\n",
"| approx_kl | 0.0054131867 |\n",
"| clip_fraction | 0.0503 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.303 |\n",
"| explained_variance | -5.86 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0238 |\n",
"| n_updates | 240 |\n",
"| policy_gradient_loss | -0.00928 |\n",
"| value_loss | 1.5e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 26 |\n",
"| time_elapsed | 105 |\n",
"| total_timesteps | 53248 |\n",
"| train/ | |\n",
"| approx_kl | 0.0050631175 |\n",
"| clip_fraction | 0.0424 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.313 |\n",
"| explained_variance | -7.31 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0182 |\n",
"| n_updates | 250 |\n",
"| policy_gradient_loss | -0.00901 |\n",
"| value_loss | 0.000127 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 27 |\n",
"| time_elapsed | 109 |\n",
"| total_timesteps | 55296 |\n",
"| train/ | |\n",
"| approx_kl | 0.0026903055 |\n",
"| clip_fraction | 0.0326 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.291 |\n",
"| explained_variance | -10 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.00649 |\n",
"| n_updates | 260 |\n",
"| policy_gradient_loss | -0.00568 |\n",
"| value_loss | 2.96e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 28 |\n",
"| time_elapsed | 113 |\n",
"| total_timesteps | 57344 |\n",
"| train/ | |\n",
"| approx_kl | 0.0036246267 |\n",
"| clip_fraction | 0.0355 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.271 |\n",
"| explained_variance | -9.29 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00148 |\n",
"| n_updates | 270 |\n",
"| policy_gradient_loss | -0.00581 |\n",
"| value_loss | 4.05e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 505 |\n",
"| iterations | 29 |\n",
"| time_elapsed | 117 |\n",
"| total_timesteps | 59392 |\n",
"| train/ | |\n",
"| approx_kl | 0.0024851589 |\n",
"| clip_fraction | 0.0375 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.284 |\n",
"| explained_variance | -5.18 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0342 |\n",
"| n_updates | 280 |\n",
"| policy_gradient_loss | -0.00556 |\n",
"| value_loss | 1.29e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 499 |\n",
"| iterations | 30 |\n",
"| time_elapsed | 122 |\n",
"| total_timesteps | 61440 |\n",
"| train/ | |\n",
"| approx_kl | 0.0028119227 |\n",
"| clip_fraction | 0.0378 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.276 |\n",
"| explained_variance | -3.17 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0172 |\n",
"| n_updates | 290 |\n",
"| policy_gradient_loss | -0.00783 |\n",
"| value_loss | 3.1e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 484 |\n",
"| iterations | 31 |\n",
"| time_elapsed | 131 |\n",
"| total_timesteps | 63488 |\n",
"| train/ | |\n",
"| approx_kl | 0.0064774947 |\n",
"| clip_fraction | 0.0411 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.249 |\n",
"| explained_variance | -8.33 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0163 |\n",
"| n_updates | 300 |\n",
"| policy_gradient_loss | -0.00733 |\n",
"| value_loss | 5.18e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 472 |\n",
"| iterations | 32 |\n",
"| time_elapsed | 138 |\n",
"| total_timesteps | 65536 |\n",
"| train/ | |\n",
"| approx_kl | 0.0043396214 |\n",
"| clip_fraction | 0.0306 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.258 |\n",
"| explained_variance | -5 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0161 |\n",
"| n_updates | 310 |\n",
"| policy_gradient_loss | -0.00749 |\n",
"| value_loss | 0.000281 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 467 |\n",
"| iterations | 33 |\n",
"| time_elapsed | 144 |\n",
"| total_timesteps | 67584 |\n",
"| train/ | |\n",
"| approx_kl | 0.0057827905 |\n",
"| clip_fraction | 0.0433 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.239 |\n",
"| explained_variance | -9 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0578 |\n",
"| n_updates | 320 |\n",
"| policy_gradient_loss | -0.0075 |\n",
"| value_loss | 8.34e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 451 |\n",
"| iterations | 34 |\n",
"| time_elapsed | 154 |\n",
"| total_timesteps | 69632 |\n",
"| train/ | |\n",
"| approx_kl | 0.0021125488 |\n",
"| clip_fraction | 0.0399 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.182 |\n",
"| explained_variance | -3.59 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0116 |\n",
"| n_updates | 330 |\n",
"| policy_gradient_loss | -0.00369 |\n",
"| value_loss | 1.17e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 441 |\n",
"| iterations | 35 |\n",
"| time_elapsed | 162 |\n",
"| total_timesteps | 71680 |\n",
"| train/ | |\n",
"| approx_kl | 0.0020559407 |\n",
"| clip_fraction | 0.0153 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.19 |\n",
"| explained_variance | -3.77 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00623 |\n",
"| n_updates | 340 |\n",
"| policy_gradient_loss | -0.00447 |\n",
"| value_loss | 2.41e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 432 |\n",
"| iterations | 36 |\n",
"| time_elapsed | 170 |\n",
"| total_timesteps | 73728 |\n",
"| train/ | |\n",
"| approx_kl | 0.0029917397 |\n",
"| clip_fraction | 0.0263 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.178 |\n",
"| explained_variance | -4.57 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0217 |\n",
"| n_updates | 350 |\n",
"| policy_gradient_loss | -0.0037 |\n",
"| value_loss | 2.88e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 426 |\n",
"| iterations | 37 |\n",
"| time_elapsed | 177 |\n",
"| total_timesteps | 75776 |\n",
"| train/ | |\n",
"| approx_kl | 0.0024150917 |\n",
"| clip_fraction | 0.0241 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.138 |\n",
"| explained_variance | -4.13 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0247 |\n",
"| n_updates | 360 |\n",
"| policy_gradient_loss | -0.00457 |\n",
"| value_loss | 5.44e-05 |\n",
"------------------------------------------\n",
"-------------------------------------------\n",
"| time/ | |\n",
"| fps | 420 |\n",
"| iterations | 38 |\n",
"| time_elapsed | 185 |\n",
"| total_timesteps | 77824 |\n",
"| train/ | |\n",
"| approx_kl | 0.00088574155 |\n",
"| clip_fraction | 0.0103 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.102 |\n",
"| explained_variance | -2.32 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0483 |\n",
"| n_updates | 370 |\n",
"| policy_gradient_loss | -0.00162 |\n",
"| value_loss | 9.93e-05 |\n",
"-------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 410 |\n",
"| iterations | 39 |\n",
"| time_elapsed | 194 |\n",
"| total_timesteps | 79872 |\n",
"| train/ | |\n",
"| approx_kl | 0.0015186302 |\n",
"| clip_fraction | 0.0154 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.115 |\n",
"| explained_variance | -4.36 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.00566 |\n",
"| n_updates | 380 |\n",
"| policy_gradient_loss | -0.00212 |\n",
"| value_loss | 1.36e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 404 |\n",
"| iterations | 40 |\n",
"| time_elapsed | 202 |\n",
"| total_timesteps | 81920 |\n",
"| train/ | |\n",
"| approx_kl | 0.0010117381 |\n",
"| clip_fraction | 0.0147 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.0774 |\n",
"| explained_variance | -13.6 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0186 |\n",
"| n_updates | 390 |\n",
"| policy_gradient_loss | -0.00316 |\n",
"| value_loss | 1.99e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 400 |\n",
"| iterations | 41 |\n",
"| time_elapsed | 209 |\n",
"| total_timesteps | 83968 |\n",
"| train/ | |\n",
"| approx_kl | 0.0011792822 |\n",
"| clip_fraction | 0.013 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.103 |\n",
"| explained_variance | -2.33 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 8.65e-05 |\n",
"| n_updates | 400 |\n",
"| policy_gradient_loss | -0.00386 |\n",
"| value_loss | 9.81e-05 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 395 |\n",
"| iterations | 42 |\n",
"| time_elapsed | 217 |\n",
"| total_timesteps | 86016 |\n",
"| train/ | |\n",
"| approx_kl | 0.0017598666 |\n",
"| clip_fraction | 0.0228 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.103 |\n",
"| explained_variance | -6.97 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0242 |\n",
"| n_updates | 410 |\n",
"| policy_gradient_loss | -0.00448 |\n",
"| value_loss | 2.37e-05 |\n",
"------------------------------------------\n",
"----------------------------------------\n",
"| time/ | |\n",
"| fps | 388 |\n",
"| iterations | 43 |\n",
"| time_elapsed | 226 |\n",
"| total_timesteps | 88064 |\n",
"| train/ | |\n",
"| approx_kl | 0.00175269 |\n",
"| clip_fraction | 0.0197 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.144 |\n",
"| explained_variance | -1.69 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0162 |\n",
"| n_updates | 420 |\n",
"| policy_gradient_loss | -0.00368 |\n",
"| value_loss | 9.39e-06 |\n",
"----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 381 |\n",
"| iterations | 44 |\n",
"| time_elapsed | 236 |\n",
"| total_timesteps | 90112 |\n",
"| train/ | |\n",
"| approx_kl | 0.003029982 |\n",
"| clip_fraction | 0.0339 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.162 |\n",
"| explained_variance | -2.24 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0335 |\n",
"| n_updates | 430 |\n",
"| policy_gradient_loss | -0.00511 |\n",
"| value_loss | 8.37e-06 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 380 |\n",
"| iterations | 45 |\n",
"| time_elapsed | 242 |\n",
"| total_timesteps | 92160 |\n",
"| train/ | |\n",
"| approx_kl | 0.0027306322 |\n",
"| clip_fraction | 0.0239 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.145 |\n",
"| explained_variance | -3.44 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.008 |\n",
"| n_updates | 440 |\n",
"| policy_gradient_loss | -0.00584 |\n",
"| value_loss | 1.1e-05 |\n",
"------------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 375 |\n",
"| iterations | 46 |\n",
"| time_elapsed | 250 |\n",
"| total_timesteps | 94208 |\n",
"| train/ | |\n",
"| approx_kl | 0.003130577 |\n",
"| clip_fraction | 0.0269 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.162 |\n",
"| explained_variance | -3.31 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0218 |\n",
"| n_updates | 450 |\n",
"| policy_gradient_loss | -0.00694 |\n",
"| value_loss | 0.000233 |\n",
"-----------------------------------------\n",
"-----------------------------------------\n",
"| time/ | |\n",
"| fps | 371 |\n",
"| iterations | 47 |\n",
"| time_elapsed | 259 |\n",
"| total_timesteps | 96256 |\n",
"| train/ | |\n",
"| approx_kl | 0.005973259 |\n",
"| clip_fraction | 0.044 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.145 |\n",
"| explained_variance | -6.63 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | 0.0204 |\n",
"| n_updates | 460 |\n",
"| policy_gradient_loss | -0.0145 |\n",
"| value_loss | 1.8e-05 |\n",
"-----------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 368 |\n",
"| iterations | 48 |\n",
"| time_elapsed | 267 |\n",
"| total_timesteps | 98304 |\n",
"| train/ | |\n",
"| approx_kl | 0.0028656125 |\n",
"| clip_fraction | 0.0293 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.152 |\n",
"| explained_variance | -2.07 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.0346 |\n",
"| n_updates | 470 |\n",
"| policy_gradient_loss | -0.00659 |\n",
"| value_loss | 7.13e-06 |\n",
"------------------------------------------\n",
"------------------------------------------\n",
"| time/ | |\n",
"| fps | 368 |\n",
"| iterations | 49 |\n",
"| time_elapsed | 272 |\n",
"| total_timesteps | 100352 |\n",
"| train/ | |\n",
"| approx_kl | 0.0018704929 |\n",
"| clip_fraction | 0.0213 |\n",
"| clip_range | 0.2 |\n",
"| entropy_loss | -0.14 |\n",
"| explained_variance | -3.72 |\n",
"| learning_rate | 0.0003 |\n",
"| loss | -0.00298 |\n",
"| n_updates | 480 |\n",
"| policy_gradient_loss | -0.00328 |\n",
"| value_loss | 1.33e-05 |\n",
"------------------------------------------\n",
"Saved model to: G:\\My Drive\\Bots DRL\\DRL\\DRL-MT5-Lab\\notebooks\\models\\ppo_EURUSD_M15.zip\n"
]
}
],
"source": [
"\n",
"# Train PPO\n",
"import numpy as np\n",
"from stable_baselines3.common.env_util import make_vec_env\n",
"\n",
"def make_env():\n",
" return TradingEnv(df_train, feature_cols)\n",
"\n",
"env = DummyVecEnv([make_env])\n",
"model = PPO(\"MlpPolicy\", env, verbose=1)\n",
"\n",
"total_timesteps = int(os.getenv(\"TOTAL_TIMESTEPS\", \"100000\"))\n",
"model.learn(total_timesteps=total_timesteps)\n",
"\n",
"Path(\"models\").mkdir(exist_ok=True)\n",
"model_path = Path(\"models\") / f\"ppo_{SYMBOL}_{TIMEFRAME}.zip\"\n",
"model.save(model_path.as_posix())\n",
"\n",
"with open(Path(\"models\") / \"selected_features.json\", \"w\", encoding=\"utf-8\") as f:\n",
" json.dump(feature_cols, f, indent=2)\n",
"\n",
"print(\"Saved model to:\", model_path.resolve())"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "cf3bf8dd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cpu device\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 359 |\n",
"| iterations | 100 |\n",
"| time_elapsed | 1 |\n",
"| total_timesteps | 500 |\n",
"| train/ | |\n",
"| entropy_loss | -1.05 |\n",
"| explained_variance | -12.1 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 99 |\n",
"| policy_loss | 0.00607 |\n",
"| value_loss | 5.1e-05 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 362 |\n",
"| iterations | 200 |\n",
"| time_elapsed | 2 |\n",
"| total_timesteps | 1000 |\n",
"| train/ | |\n",
"| entropy_loss | -1.05 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 199 |\n",
"| policy_loss | -0.00202 |\n",
"| value_loss | 3.28e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 370 |\n",
"| iterations | 300 |\n",
"| time_elapsed | 4 |\n",
"| total_timesteps | 1500 |\n",
"| train/ | |\n",
"| entropy_loss | -1.02 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 299 |\n",
"| policy_loss | -0.000347 |\n",
"| value_loss | 1.86e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 352 |\n",
"| iterations | 400 |\n",
"| time_elapsed | 5 |\n",
"| total_timesteps | 2000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.896 |\n",
"| explained_variance | 0.0136 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 399 |\n",
"| policy_loss | 0.00511 |\n",
"| value_loss | 2.97e-05 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 345 |\n",
"| iterations | 500 |\n",
"| time_elapsed | 7 |\n",
"| total_timesteps | 2500 |\n",
"| train/ | |\n",
"| entropy_loss | -1.02 |\n",
"| explained_variance | -1.63 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 499 |\n",
"| policy_loss | 0.0098 |\n",
"| value_loss | 0.000109 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 341 |\n",
"| iterations | 600 |\n",
"| time_elapsed | 8 |\n",
"| total_timesteps | 3000 |\n",
"| train/ | |\n",
"| entropy_loss | -1.04 |\n",
"| explained_variance | -64.9 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 599 |\n",
"| policy_loss | 0.00834 |\n",
"| value_loss | 0.000233 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 331 |\n",
"| iterations | 700 |\n",
"| time_elapsed | 10 |\n",
"| total_timesteps | 3500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.92 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 699 |\n",
"| policy_loss | 0.000506 |\n",
"| value_loss | 2.37e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 328 |\n",
"| iterations | 800 |\n",
"| time_elapsed | 12 |\n",
"| total_timesteps | 4000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.941 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 799 |\n",
"| policy_loss | 0.000532 |\n",
"| value_loss | 3.67e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 326 |\n",
"| iterations | 900 |\n",
"| time_elapsed | 13 |\n",
"| total_timesteps | 4500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.851 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 899 |\n",
"| policy_loss | -0.000172 |\n",
"| value_loss | 2.84e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 1000 |\n",
"| time_elapsed | 15 |\n",
"| total_timesteps | 5000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.599 |\n",
"| explained_variance | -4.26 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 999 |\n",
"| policy_loss | 0.00652 |\n",
"| value_loss | 7.48e-05 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 1100 |\n",
"| time_elapsed | 17 |\n",
"| total_timesteps | 5500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.598 |\n",
"| explained_variance | -67.3 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1099 |\n",
"| policy_loss | -0.00226 |\n",
"| value_loss | 5.66e-05 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 1200 |\n",
"| time_elapsed | 18 |\n",
"| total_timesteps | 6000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.6 |\n",
"| explained_variance | -0.0421 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1199 |\n",
"| policy_loss | -0.000824 |\n",
"| value_loss | 6.37e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 1300 |\n",
"| time_elapsed | 20 |\n",
"| total_timesteps | 6500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.96 |\n",
"| explained_variance | -8.23e+03 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1299 |\n",
"| policy_loss | -0.0152 |\n",
"| value_loss | 0.00129 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 326 |\n",
"| iterations | 1400 |\n",
"| time_elapsed | 21 |\n",
"| total_timesteps | 7000 |\n",
"| train/ | |\n",
"| entropy_loss | -1 |\n",
"| explained_variance | -1.37e+03 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1399 |\n",
"| policy_loss | 0.00168 |\n",
"| value_loss | 1.88e-05 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 328 |\n",
"| iterations | 1500 |\n",
"| time_elapsed | 22 |\n",
"| total_timesteps | 7500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.737 |\n",
"| explained_variance | -3.79 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1499 |\n",
"| policy_loss | 0.00252 |\n",
"| value_loss | 1.9e-05 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 330 |\n",
"| iterations | 1600 |\n",
"| time_elapsed | 24 |\n",
"| total_timesteps | 8000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.657 |\n",
"| explained_variance | -43.7 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1599 |\n",
"| policy_loss | 0.0008 |\n",
"| value_loss | 1.34e-05 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 332 |\n",
"| iterations | 1700 |\n",
"| time_elapsed | 25 |\n",
"| total_timesteps | 8500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.63 |\n",
"| explained_variance | -7.88 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1699 |\n",
"| policy_loss | -0.000272 |\n",
"| value_loss | 1.75e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 332 |\n",
"| iterations | 1800 |\n",
"| time_elapsed | 27 |\n",
"| total_timesteps | 9000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.531 |\n",
"| explained_variance | -159 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1799 |\n",
"| policy_loss | -0.00301 |\n",
"| value_loss | 0.000184 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 331 |\n",
"| iterations | 1900 |\n",
"| time_elapsed | 28 |\n",
"| total_timesteps | 9500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.561 |\n",
"| explained_variance | -3.84e+05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1899 |\n",
"| policy_loss | 0.00499 |\n",
"| value_loss | 0.0105 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 329 |\n",
"| iterations | 2000 |\n",
"| time_elapsed | 30 |\n",
"| total_timesteps | 10000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.377 |\n",
"| explained_variance | -894 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 1999 |\n",
"| policy_loss | -0.00021 |\n",
"| value_loss | 8.1e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 328 |\n",
"| iterations | 2100 |\n",
"| time_elapsed | 31 |\n",
"| total_timesteps | 10500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.357 |\n",
"| explained_variance | 0.00573 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2099 |\n",
"| policy_loss | -0.000193 |\n",
"| value_loss | 4.08e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 328 |\n",
"| iterations | 2200 |\n",
"| time_elapsed | 33 |\n",
"| total_timesteps | 11000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.196 |\n",
"| explained_variance | -57 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2199 |\n",
"| policy_loss | 0.000124 |\n",
"| value_loss | 8.03e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 326 |\n",
"| iterations | 2300 |\n",
"| time_elapsed | 35 |\n",
"| total_timesteps | 11500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0896 |\n",
"| explained_variance | -139 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2299 |\n",
"| policy_loss | 2.65e-05 |\n",
"| value_loss | 6.23e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 2400 |\n",
"| time_elapsed | 36 |\n",
"| total_timesteps | 12000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.302 |\n",
"| explained_variance | -58.3 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2399 |\n",
"| policy_loss | 0.000408 |\n",
"| value_loss | 7.1e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 2500 |\n",
"| time_elapsed | 38 |\n",
"| total_timesteps | 12500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0701 |\n",
"| explained_variance | 0.172 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2499 |\n",
"| policy_loss | 3.23e-08 |\n",
"| value_loss | 1.81e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 2600 |\n",
"| time_elapsed | 40 |\n",
"| total_timesteps | 13000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0533 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2599 |\n",
"| policy_loss | -9.45e-07 |\n",
"| value_loss | 4.53e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 2700 |\n",
"| time_elapsed | 41 |\n",
"| total_timesteps | 13500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0479 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2699 |\n",
"| policy_loss | 4.05e-06 |\n",
"| value_loss | 3.2e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 2800 |\n",
"| time_elapsed | 43 |\n",
"| total_timesteps | 14000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0477 |\n",
"| explained_variance | -0.807 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2799 |\n",
"| policy_loss | -9.65e-07 |\n",
"| value_loss | 3.52e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 2900 |\n",
"| time_elapsed | 45 |\n",
"| total_timesteps | 14500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.141 |\n",
"| explained_variance | -33.4 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2899 |\n",
"| policy_loss | -9.53e-05 |\n",
"| value_loss | 5.38e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 319 |\n",
"| iterations | 3000 |\n",
"| time_elapsed | 46 |\n",
"| total_timesteps | 15000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0618 |\n",
"| explained_variance | -0.102 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 2999 |\n",
"| policy_loss | -2.49e-06 |\n",
"| value_loss | 7.56e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 319 |\n",
"| iterations | 3100 |\n",
"| time_elapsed | 48 |\n",
"| total_timesteps | 15500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0417 |\n",
"| explained_variance | -0.0284 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3099 |\n",
"| policy_loss | -9.97e-06 |\n",
"| value_loss | 3.2e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 319 |\n",
"| iterations | 3200 |\n",
"| time_elapsed | 50 |\n",
"| total_timesteps | 16000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0643 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3199 |\n",
"| policy_loss | 1.16e-05 |\n",
"| value_loss | 1.58e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 3300 |\n",
"| time_elapsed | 51 |\n",
"| total_timesteps | 16500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.061 |\n",
"| explained_variance | -6.01 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3299 |\n",
"| policy_loss | -2.23e-05 |\n",
"| value_loss | 5.67e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 3400 |\n",
"| time_elapsed | 52 |\n",
"| total_timesteps | 17000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0524 |\n",
"| explained_variance | 0.349 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3399 |\n",
"| policy_loss | 4.4e-05 |\n",
"| value_loss | 2.86e-05 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 3500 |\n",
"| time_elapsed | 54 |\n",
"| total_timesteps | 17500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0491 |\n",
"| explained_variance | -30.4 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3499 |\n",
"| policy_loss | -2.74e-05 |\n",
"| value_loss | 1.57e-05 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 3600 |\n",
"| time_elapsed | 55 |\n",
"| total_timesteps | 18000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0216 |\n",
"| explained_variance | 0.00218 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3599 |\n",
"| policy_loss | -1.1e-06 |\n",
"| value_loss | 2.05e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 3700 |\n",
"| time_elapsed | 57 |\n",
"| total_timesteps | 18500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0218 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3699 |\n",
"| policy_loss | -5.02e-07 |\n",
"| value_loss | 3.1e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 3800 |\n",
"| time_elapsed | 58 |\n",
"| total_timesteps | 19000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.024 |\n",
"| explained_variance | -4.98 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3799 |\n",
"| policy_loss | -5.68e-06 |\n",
"| value_loss | 4.92e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 3900 |\n",
"| time_elapsed | 60 |\n",
"| total_timesteps | 19500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0226 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3899 |\n",
"| policy_loss | 4.01e-07 |\n",
"| value_loss | 6.52e-08 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 4000 |\n",
"| time_elapsed | 62 |\n",
"| total_timesteps | 20000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0181 |\n",
"| explained_variance | -1.72e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 3999 |\n",
"| policy_loss | 1.36e-06 |\n",
"| value_loss | 3.18e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 4100 |\n",
"| time_elapsed | 63 |\n",
"| total_timesteps | 20500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0168 |\n",
"| explained_variance | 5.96e-08 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4099 |\n",
"| policy_loss | 1.98e-06 |\n",
"| value_loss | 8.21e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 4200 |\n",
"| time_elapsed | 65 |\n",
"| total_timesteps | 21000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0137 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4199 |\n",
"| policy_loss | -1.08e-06 |\n",
"| value_loss | 5.07e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 4300 |\n",
"| time_elapsed | 66 |\n",
"| total_timesteps | 21500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0155 |\n",
"| explained_variance | -24.2 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4299 |\n",
"| policy_loss | 0.00232 |\n",
"| value_loss | 3.59e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 4400 |\n",
"| time_elapsed | 68 |\n",
"| total_timesteps | 22000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0163 |\n",
"| explained_variance | -1.56 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4399 |\n",
"| policy_loss | 5.63e-07 |\n",
"| value_loss | 1.49e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 4500 |\n",
"| time_elapsed | 70 |\n",
"| total_timesteps | 22500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0176 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4499 |\n",
"| policy_loss | 7.63e-07 |\n",
"| value_loss | 2.56e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 4600 |\n",
"| time_elapsed | 71 |\n",
"| total_timesteps | 23000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0185 |\n",
"| explained_variance | 5.96e-08 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4599 |\n",
"| policy_loss | 3.89e-06 |\n",
"| value_loss | 2.59e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 4700 |\n",
"| time_elapsed | 73 |\n",
"| total_timesteps | 23500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0184 |\n",
"| explained_variance | 5.96e-08 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4699 |\n",
"| policy_loss | -5.51e-07 |\n",
"| value_loss | 2.17e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 4800 |\n",
"| time_elapsed | 74 |\n",
"| total_timesteps | 24000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0183 |\n",
"| explained_variance | -5.05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4799 |\n",
"| policy_loss | -1.53e-06 |\n",
"| value_loss | 6.76e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 4900 |\n",
"| time_elapsed | 76 |\n",
"| total_timesteps | 24500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.016 |\n",
"| explained_variance | -2.41 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4899 |\n",
"| policy_loss | -1.9e-06 |\n",
"| value_loss | 2.45e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 319 |\n",
"| iterations | 5000 |\n",
"| time_elapsed | 78 |\n",
"| total_timesteps | 25000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.02 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 4999 |\n",
"| policy_loss | -1.46e-06 |\n",
"| value_loss | 6.88e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 320 |\n",
"| iterations | 5100 |\n",
"| time_elapsed | 79 |\n",
"| total_timesteps | 25500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.021 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5099 |\n",
"| policy_loss | -3.64e-06 |\n",
"| value_loss | 1.73e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 321 |\n",
"| iterations | 5200 |\n",
"| time_elapsed | 80 |\n",
"| total_timesteps | 26000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0193 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5199 |\n",
"| policy_loss | 6.82e-07 |\n",
"| value_loss | 9.1e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 322 |\n",
"| iterations | 5300 |\n",
"| time_elapsed | 82 |\n",
"| total_timesteps | 26500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.019 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5299 |\n",
"| policy_loss | 6.54e-07 |\n",
"| value_loss | 9.62e-08 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 5400 |\n",
"| time_elapsed | 83 |\n",
"| total_timesteps | 27000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0312 |\n",
"| explained_variance | 2.99e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5399 |\n",
"| policy_loss | -1.18e-06 |\n",
"| value_loss | 9.25e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 5500 |\n",
"| time_elapsed | 84 |\n",
"| total_timesteps | 27500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0149 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5499 |\n",
"| policy_loss | -1.13e-06 |\n",
"| value_loss | 3.87e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 324 |\n",
"| iterations | 5600 |\n",
"| time_elapsed | 86 |\n",
"| total_timesteps | 28000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0142 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5599 |\n",
"| policy_loss | 8.31e-07 |\n",
"| value_loss | 2.3e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 5700 |\n",
"| time_elapsed | 87 |\n",
"| total_timesteps | 28500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0139 |\n",
"| explained_variance | -10.4 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5699 |\n",
"| policy_loss | -3.13e-06 |\n",
"| value_loss | 4.23e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 5800 |\n",
"| time_elapsed | 89 |\n",
"| total_timesteps | 29000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0133 |\n",
"| explained_variance | -6.01e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5799 |\n",
"| policy_loss | 1.89e-07 |\n",
"| value_loss | 1.56e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 323 |\n",
"| iterations | 5900 |\n",
"| time_elapsed | 91 |\n",
"| total_timesteps | 29500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.013 |\n",
"| explained_variance | -6.75 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5899 |\n",
"| policy_loss | 1.93e-06 |\n",
"| value_loss | 2.21e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 319 |\n",
"| iterations | 6000 |\n",
"| time_elapsed | 93 |\n",
"| total_timesteps | 30000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0156 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 5999 |\n",
"| policy_loss | 1.19e-06 |\n",
"| value_loss | 3.89e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 316 |\n",
"| iterations | 6100 |\n",
"| time_elapsed | 96 |\n",
"| total_timesteps | 30500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0217 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6099 |\n",
"| policy_loss | -8.86e-05 |\n",
"| value_loss | 3.24e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 313 |\n",
"| iterations | 6200 |\n",
"| time_elapsed | 98 |\n",
"| total_timesteps | 31000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0288 |\n",
"| explained_variance | -18.4 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6199 |\n",
"| policy_loss | 1.51e-06 |\n",
"| value_loss | 2.35e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 310 |\n",
"| iterations | 6300 |\n",
"| time_elapsed | 101 |\n",
"| total_timesteps | 31500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0151 |\n",
"| explained_variance | -0.371 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6299 |\n",
"| policy_loss | 3.38e-07 |\n",
"| value_loss | 6.04e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 307 |\n",
"| iterations | 6400 |\n",
"| time_elapsed | 104 |\n",
"| total_timesteps | 32000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0108 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6399 |\n",
"| policy_loss | 1.74e-06 |\n",
"| value_loss | 1.88e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 303 |\n",
"| iterations | 6500 |\n",
"| time_elapsed | 107 |\n",
"| total_timesteps | 32500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.008 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6499 |\n",
"| policy_loss | 4.8e-07 |\n",
"| value_loss | 4.41e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 300 |\n",
"| iterations | 6600 |\n",
"| time_elapsed | 109 |\n",
"| total_timesteps | 33000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0077 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6599 |\n",
"| policy_loss | 1.41e-07 |\n",
"| value_loss | 1.94e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 296 |\n",
"| iterations | 6700 |\n",
"| time_elapsed | 112 |\n",
"| total_timesteps | 33500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00764 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6699 |\n",
"| policy_loss | 4.16e-07 |\n",
"| value_loss | 2.53e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 292 |\n",
"| iterations | 6800 |\n",
"| time_elapsed | 116 |\n",
"| total_timesteps | 34000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00738 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6799 |\n",
"| policy_loss | 1.8e-07 |\n",
"| value_loss | 5.08e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 291 |\n",
"| iterations | 6900 |\n",
"| time_elapsed | 118 |\n",
"| total_timesteps | 34500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00749 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6899 |\n",
"| policy_loss | -6.73e-07 |\n",
"| value_loss | 7.23e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 290 |\n",
"| iterations | 7000 |\n",
"| time_elapsed | 120 |\n",
"| total_timesteps | 35000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00749 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 6999 |\n",
"| policy_loss | -4.07e-07 |\n",
"| value_loss | 3.65e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 289 |\n",
"| iterations | 7100 |\n",
"| time_elapsed | 122 |\n",
"| total_timesteps | 35500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00455 |\n",
"| explained_variance | -8.14 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7099 |\n",
"| policy_loss | 2.3e-07 |\n",
"| value_loss | 4.69e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 289 |\n",
"| iterations | 7200 |\n",
"| time_elapsed | 124 |\n",
"| total_timesteps | 36000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00469 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7199 |\n",
"| policy_loss | -6.61e-08 |\n",
"| value_loss | 1.93e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 288 |\n",
"| iterations | 7300 |\n",
"| time_elapsed | 126 |\n",
"| total_timesteps | 36500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00533 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7299 |\n",
"| policy_loss | -8.97e-07 |\n",
"| value_loss | 2.61e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 288 |\n",
"| iterations | 7400 |\n",
"| time_elapsed | 128 |\n",
"| total_timesteps | 37000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00551 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7399 |\n",
"| policy_loss | -4.53e-08 |\n",
"| value_loss | 1.7e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 288 |\n",
"| iterations | 7500 |\n",
"| time_elapsed | 130 |\n",
"| total_timesteps | 37500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0055 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7499 |\n",
"| policy_loss | -1.91e-07 |\n",
"| value_loss | 1.83e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 287 |\n",
"| iterations | 7600 |\n",
"| time_elapsed | 132 |\n",
"| total_timesteps | 38000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00544 |\n",
"| explained_variance | -0.00785 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7599 |\n",
"| policy_loss | 6.53e-07 |\n",
"| value_loss | 1.41e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 287 |\n",
"| iterations | 7700 |\n",
"| time_elapsed | 133 |\n",
"| total_timesteps | 38500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00537 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7699 |\n",
"| policy_loss | 3.4e-07 |\n",
"| value_loss | 4.42e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| time/ | |\n",
"| fps | 287 |\n",
"| iterations | 7800 |\n",
"| time_elapsed | 135 |\n",
"| total_timesteps | 39000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00567 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7799 |\n",
"| policy_loss | -6.66e-09 |\n",
"| value_loss | 4.08e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| time/ | |\n",
"| fps | 286 |\n",
"| iterations | 7900 |\n",
"| time_elapsed | 137 |\n",
"| total_timesteps | 39500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00558 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7899 |\n",
"| policy_loss | -3e-07 |\n",
"| value_loss | 4.75e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 284 |\n",
"| iterations | 8000 |\n",
"| time_elapsed | 140 |\n",
"| total_timesteps | 40000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00341 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 7999 |\n",
"| policy_loss | -5.68e-07 |\n",
"| value_loss | 3.09e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 283 |\n",
"| iterations | 8100 |\n",
"| time_elapsed | 142 |\n",
"| total_timesteps | 40500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00343 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8099 |\n",
"| policy_loss | 1.61e-07 |\n",
"| value_loss | 2.61e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 283 |\n",
"| iterations | 8200 |\n",
"| time_elapsed | 144 |\n",
"| total_timesteps | 41000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00334 |\n",
"| explained_variance | 5.96e-08 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8199 |\n",
"| policy_loss | -1.11e-07 |\n",
"| value_loss | 1.81e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 282 |\n",
"| iterations | 8300 |\n",
"| time_elapsed | 146 |\n",
"| total_timesteps | 41500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00306 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8299 |\n",
"| policy_loss | 1.19e-07 |\n",
"| value_loss | 2.34e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 281 |\n",
"| iterations | 8400 |\n",
"| time_elapsed | 149 |\n",
"| total_timesteps | 42000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00274 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8399 |\n",
"| policy_loss | 1.79e-07 |\n",
"| value_loss | 5.11e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 280 |\n",
"| iterations | 8500 |\n",
"| time_elapsed | 151 |\n",
"| total_timesteps | 42500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00323 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8499 |\n",
"| policy_loss | -2.51e-07 |\n",
"| value_loss | 9.46e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 279 |\n",
"| iterations | 8600 |\n",
"| time_elapsed | 153 |\n",
"| total_timesteps | 43000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0034 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8599 |\n",
"| policy_loss | 3.08e-07 |\n",
"| value_loss | 4e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 279 |\n",
"| iterations | 8700 |\n",
"| time_elapsed | 155 |\n",
"| total_timesteps | 43500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00381 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8699 |\n",
"| policy_loss | 1.01e-07 |\n",
"| value_loss | 9.43e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 279 |\n",
"| iterations | 8800 |\n",
"| time_elapsed | 157 |\n",
"| total_timesteps | 44000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00322 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8799 |\n",
"| policy_loss | -4.56e-07 |\n",
"| value_loss | 2.21e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 278 |\n",
"| iterations | 8900 |\n",
"| time_elapsed | 159 |\n",
"| total_timesteps | 44500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00289 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8899 |\n",
"| policy_loss | -1.57e-07 |\n",
"| value_loss | 3.32e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 277 |\n",
"| iterations | 9000 |\n",
"| time_elapsed | 162 |\n",
"| total_timesteps | 45000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00307 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 8999 |\n",
"| policy_loss | 9.29e-08 |\n",
"| value_loss | 1.2e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 277 |\n",
"| iterations | 9100 |\n",
"| time_elapsed | 164 |\n",
"| total_timesteps | 45500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00318 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9099 |\n",
"| policy_loss | 3.38e-07 |\n",
"| value_loss | 1.18e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 277 |\n",
"| iterations | 9200 |\n",
"| time_elapsed | 165 |\n",
"| total_timesteps | 46000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00259 |\n",
"| explained_variance | -45.4 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9199 |\n",
"| policy_loss | 1.23e-08 |\n",
"| value_loss | 2.24e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 277 |\n",
"| iterations | 9300 |\n",
"| time_elapsed | 167 |\n",
"| total_timesteps | 46500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00279 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9299 |\n",
"| policy_loss | -1.45e-07 |\n",
"| value_loss | 3.89e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 276 |\n",
"| iterations | 9400 |\n",
"| time_elapsed | 169 |\n",
"| total_timesteps | 47000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00235 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9399 |\n",
"| policy_loss | -3.56e-07 |\n",
"| value_loss | 2.23e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 276 |\n",
"| iterations | 9500 |\n",
"| time_elapsed | 171 |\n",
"| total_timesteps | 47500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00241 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9499 |\n",
"| policy_loss | -2.54e-08 |\n",
"| value_loss | 3.33e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 276 |\n",
"| iterations | 9600 |\n",
"| time_elapsed | 173 |\n",
"| total_timesteps | 48000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00242 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9599 |\n",
"| policy_loss | 5.59e-08 |\n",
"| value_loss | 4.24e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 275 |\n",
"| iterations | 9700 |\n",
"| time_elapsed | 175 |\n",
"| total_timesteps | 48500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00674 |\n",
"| explained_variance | -4.03 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9699 |\n",
"| policy_loss | -3.96e-07 |\n",
"| value_loss | 1.78e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 275 |\n",
"| iterations | 9800 |\n",
"| time_elapsed | 178 |\n",
"| total_timesteps | 49000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00224 |\n",
"| explained_variance | 1.19e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9799 |\n",
"| policy_loss | 2.55e-07 |\n",
"| value_loss | 1.71e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 274 |\n",
"| iterations | 9900 |\n",
"| time_elapsed | 180 |\n",
"| total_timesteps | 49500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00223 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9899 |\n",
"| policy_loss | 1.56e-07 |\n",
"| value_loss | 8.44e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 273 |\n",
"| iterations | 10000 |\n",
"| time_elapsed | 182 |\n",
"| total_timesteps | 50000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00223 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 9999 |\n",
"| policy_loss | 2.31e-07 |\n",
"| value_loss | 1.16e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10100 |\n",
"| time_elapsed | 185 |\n",
"| total_timesteps | 50500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00201 |\n",
"| explained_variance | 9.94e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10099 |\n",
"| policy_loss | 6.55e-08 |\n",
"| value_loss | 1.2e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10200 |\n",
"| time_elapsed | 187 |\n",
"| total_timesteps | 51000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00163 |\n",
"| explained_variance | -0.372 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10199 |\n",
"| policy_loss | 3.4e-07 |\n",
"| value_loss | 5.94e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10300 |\n",
"| time_elapsed | 189 |\n",
"| total_timesteps | 51500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00199 |\n",
"| explained_variance | 3.44e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10299 |\n",
"| policy_loss | 3.58e-08 |\n",
"| value_loss | 5.18e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10400 |\n",
"| time_elapsed | 191 |\n",
"| total_timesteps | 52000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00196 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10399 |\n",
"| policy_loss | 2.64e-07 |\n",
"| value_loss | 2.31e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10500 |\n",
"| time_elapsed | 192 |\n",
"| total_timesteps | 52500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00241 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10499 |\n",
"| policy_loss | 6.48e-07 |\n",
"| value_loss | 7.47e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10600 |\n",
"| time_elapsed | 194 |\n",
"| total_timesteps | 53000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00177 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10599 |\n",
"| policy_loss | -5.67e-08 |\n",
"| value_loss | 1.37e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10700 |\n",
"| time_elapsed | 196 |\n",
"| total_timesteps | 53500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00176 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10699 |\n",
"| policy_loss | 7.04e-08 |\n",
"| value_loss | 1.78e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10800 |\n",
"| time_elapsed | 198 |\n",
"| total_timesteps | 54000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00177 |\n",
"| explained_variance | -2.07e-05 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10799 |\n",
"| policy_loss | 1.36e-08 |\n",
"| value_loss | 5.07e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 272 |\n",
"| iterations | 10900 |\n",
"| time_elapsed | 200 |\n",
"| total_timesteps | 54500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00199 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10899 |\n",
"| policy_loss | -2.52e-07 |\n",
"| value_loss | 1.82e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 271 |\n",
"| iterations | 11000 |\n",
"| time_elapsed | 202 |\n",
"| total_timesteps | 55000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00199 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 10999 |\n",
"| policy_loss | -8.72e-08 |\n",
"| value_loss | 4.42e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 271 |\n",
"| iterations | 11100 |\n",
"| time_elapsed | 204 |\n",
"| total_timesteps | 55500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00184 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11099 |\n",
"| policy_loss | 1.46e-08 |\n",
"| value_loss | 3.99e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 271 |\n",
"| iterations | 11200 |\n",
"| time_elapsed | 206 |\n",
"| total_timesteps | 56000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00179 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11199 |\n",
"| policy_loss | 1.79e-07 |\n",
"| value_loss | 1.24e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 270 |\n",
"| iterations | 11300 |\n",
"| time_elapsed | 208 |\n",
"| total_timesteps | 56500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00179 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11299 |\n",
"| policy_loss | -1.3e-07 |\n",
"| value_loss | 5.51e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 270 |\n",
"| iterations | 11400 |\n",
"| time_elapsed | 210 |\n",
"| total_timesteps | 57000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00192 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11399 |\n",
"| policy_loss | 4.14e-08 |\n",
"| value_loss | 2.51e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 270 |\n",
"| iterations | 11500 |\n",
"| time_elapsed | 212 |\n",
"| total_timesteps | 57500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00185 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11499 |\n",
"| policy_loss | 1.43e-07 |\n",
"| value_loss | 6.49e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 11600 |\n",
"| time_elapsed | 214 |\n",
"| total_timesteps | 58000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00528 |\n",
"| explained_variance | -0.607 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11599 |\n",
"| policy_loss | 4.24e-07 |\n",
"| value_loss | 1.18e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 11700 |\n",
"| time_elapsed | 216 |\n",
"| total_timesteps | 58500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00188 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11699 |\n",
"| policy_loss | -2.21e-07 |\n",
"| value_loss | 1.57e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 11800 |\n",
"| time_elapsed | 218 |\n",
"| total_timesteps | 59000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00189 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11799 |\n",
"| policy_loss | -5.29e-08 |\n",
"| value_loss | 1.73e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 11900 |\n",
"| time_elapsed | 220 |\n",
"| total_timesteps | 59500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00189 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11899 |\n",
"| policy_loss | 7.49e-09 |\n",
"| value_loss | 5.51e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12000 |\n",
"| time_elapsed | 222 |\n",
"| total_timesteps | 60000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00193 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 11999 |\n",
"| policy_loss | 2.09e-07 |\n",
"| value_loss | 1.2e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12100 |\n",
"| time_elapsed | 224 |\n",
"| total_timesteps | 60500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00155 |\n",
"| explained_variance | -0.000509 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12099 |\n",
"| policy_loss | 1.02e-08 |\n",
"| value_loss | 5.74e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12200 |\n",
"| time_elapsed | 226 |\n",
"| total_timesteps | 61000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0015 |\n",
"| explained_variance | 0.0013 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12199 |\n",
"| policy_loss | -3.54e-08 |\n",
"| value_loss | 6.87e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12300 |\n",
"| time_elapsed | 228 |\n",
"| total_timesteps | 61500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00197 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12299 |\n",
"| policy_loss | 4.29e-07 |\n",
"| value_loss | 5.15e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12400 |\n",
"| time_elapsed | 229 |\n",
"| total_timesteps | 62000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00196 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12399 |\n",
"| policy_loss | 1.13e-07 |\n",
"| value_loss | 4.86e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12500 |\n",
"| time_elapsed | 231 |\n",
"| total_timesteps | 62500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00196 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12499 |\n",
"| policy_loss | -3.7e-08 |\n",
"| value_loss | 7.61e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12600 |\n",
"| time_elapsed | 233 |\n",
"| total_timesteps | 63000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00308 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12599 |\n",
"| policy_loss | 2.54e-07 |\n",
"| value_loss | 6.8e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12700 |\n",
"| time_elapsed | 235 |\n",
"| total_timesteps | 63500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0029 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12699 |\n",
"| policy_loss | -2.17e-08 |\n",
"| value_loss | 2.25e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 269 |\n",
"| iterations | 12800 |\n",
"| time_elapsed | 237 |\n",
"| total_timesteps | 64000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00306 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12799 |\n",
"| policy_loss | 3.1e-07 |\n",
"| value_loss | 1.18e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 12900 |\n",
"| time_elapsed | 239 |\n",
"| total_timesteps | 64500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00327 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12899 |\n",
"| policy_loss | -9.89e-08 |\n",
"| value_loss | 3.26e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13000 |\n",
"| time_elapsed | 241 |\n",
"| total_timesteps | 65000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00329 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 12999 |\n",
"| policy_loss | 2.77e-08 |\n",
"| value_loss | 9.15e-08 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13100 |\n",
"| time_elapsed | 243 |\n",
"| total_timesteps | 65500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00334 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13099 |\n",
"| policy_loss | -2.29e-07 |\n",
"| value_loss | 6.52e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13200 |\n",
"| time_elapsed | 246 |\n",
"| total_timesteps | 66000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00334 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13199 |\n",
"| policy_loss | 1.73e-07 |\n",
"| value_loss | 2.65e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13300 |\n",
"| time_elapsed | 248 |\n",
"| total_timesteps | 66500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00334 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13299 |\n",
"| policy_loss | 5.51e-07 |\n",
"| value_loss | 3.01e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13400 |\n",
"| time_elapsed | 249 |\n",
"| total_timesteps | 67000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00334 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13399 |\n",
"| policy_loss | 1.97e-07 |\n",
"| value_loss | 4.24e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13500 |\n",
"| time_elapsed | 251 |\n",
"| total_timesteps | 67500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.039 |\n",
"| explained_variance | -26.7 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13499 |\n",
"| policy_loss | -1.84e-06 |\n",
"| value_loss | 8.37e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 268 |\n",
"| iterations | 13600 |\n",
"| time_elapsed | 253 |\n",
"| total_timesteps | 68000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00405 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13599 |\n",
"| policy_loss | -2.37e-07 |\n",
"| value_loss | 3.07e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 267 |\n",
"| iterations | 13700 |\n",
"| time_elapsed | 255 |\n",
"| total_timesteps | 68500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0041 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13699 |\n",
"| policy_loss | 1.88e-07 |\n",
"| value_loss | 2.35e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 267 |\n",
"| iterations | 13800 |\n",
"| time_elapsed | 258 |\n",
"| total_timesteps | 69000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0039 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13799 |\n",
"| policy_loss | 3e-08 |\n",
"| value_loss | 5.83e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 266 |\n",
"| iterations | 13900 |\n",
"| time_elapsed | 260 |\n",
"| total_timesteps | 69500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0039 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13899 |\n",
"| policy_loss | 5.74e-08 |\n",
"| value_loss | 3.5e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 266 |\n",
"| iterations | 14000 |\n",
"| time_elapsed | 262 |\n",
"| total_timesteps | 70000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.339 |\n",
"| explained_variance | -2.68 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 13999 |\n",
"| policy_loss | -0.00017 |\n",
"| value_loss | 2.91e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 266 |\n",
"| iterations | 14100 |\n",
"| time_elapsed | 264 |\n",
"| total_timesteps | 70500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00242 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14099 |\n",
"| policy_loss | -8.7e-08 |\n",
"| value_loss | 1.49e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 14200 |\n",
"| time_elapsed | 266 |\n",
"| total_timesteps | 71000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00264 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14199 |\n",
"| policy_loss | -1.67e-07 |\n",
"| value_loss | 4.86e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 14300 |\n",
"| time_elapsed | 269 |\n",
"| total_timesteps | 71500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00252 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14299 |\n",
"| policy_loss | -2.56e-07 |\n",
"| value_loss | 1.09e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 14400 |\n",
"| time_elapsed | 271 |\n",
"| total_timesteps | 72000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00227 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14399 |\n",
"| policy_loss | -2.36e-07 |\n",
"| value_loss | 1.11e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 14500 |\n",
"| time_elapsed | 273 |\n",
"| total_timesteps | 72500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00247 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14499 |\n",
"| policy_loss | -1.91e-07 |\n",
"| value_loss | 5.9e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 14600 |\n",
"| time_elapsed | 275 |\n",
"| total_timesteps | 73000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00272 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14599 |\n",
"| policy_loss | 1.74e-07 |\n",
"| value_loss | 4.13e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 14700 |\n",
"| time_elapsed | 277 |\n",
"| total_timesteps | 73500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00286 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14699 |\n",
"| policy_loss | -1.16e-07 |\n",
"| value_loss | 1.55e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 14800 |\n",
"| time_elapsed | 279 |\n",
"| total_timesteps | 74000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00286 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14799 |\n",
"| policy_loss | 1.03e-07 |\n",
"| value_loss | 1.67e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 14900 |\n",
"| time_elapsed | 281 |\n",
"| total_timesteps | 74500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00252 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14899 |\n",
"| policy_loss | 1.68e-07 |\n",
"| value_loss | 5e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15000 |\n",
"| time_elapsed | 282 |\n",
"| total_timesteps | 75000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00282 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 14999 |\n",
"| policy_loss | -1.24e-07 |\n",
"| value_loss | 1.88e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 15100 |\n",
"| time_elapsed | 284 |\n",
"| total_timesteps | 75500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00245 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15099 |\n",
"| policy_loss | 2.49e-08 |\n",
"| value_loss | 4.17e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15200 |\n",
"| time_elapsed | 286 |\n",
"| total_timesteps | 76000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0025 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15199 |\n",
"| policy_loss | -7.05e-08 |\n",
"| value_loss | 1.74e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15300 |\n",
"| time_elapsed | 288 |\n",
"| total_timesteps | 76500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00231 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15299 |\n",
"| policy_loss | -6.79e-08 |\n",
"| value_loss | 1.08e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15400 |\n",
"| time_elapsed | 290 |\n",
"| total_timesteps | 77000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00212 |\n",
"| explained_variance | -0.00223 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15399 |\n",
"| policy_loss | 6.95e-08 |\n",
"| value_loss | 1.41e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15500 |\n",
"| time_elapsed | 292 |\n",
"| total_timesteps | 77500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00218 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15499 |\n",
"| policy_loss | -2.11e-08 |\n",
"| value_loss | 1.85e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 265 |\n",
"| iterations | 15600 |\n",
"| time_elapsed | 294 |\n",
"| total_timesteps | 78000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00224 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15599 |\n",
"| policy_loss | -4.52e-08 |\n",
"| value_loss | 5.43e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 15700 |\n",
"| time_elapsed | 296 |\n",
"| total_timesteps | 78500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00224 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15699 |\n",
"| policy_loss | 2.38e-08 |\n",
"| value_loss | 1.14e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.548 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 15800 |\n",
"| time_elapsed | 298 |\n",
"| total_timesteps | 79000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00225 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15799 |\n",
"| policy_loss | -1.93e-08 |\n",
"| value_loss | 2.79e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 264 |\n",
"| iterations | 15900 |\n",
"| time_elapsed | 300 |\n",
"| total_timesteps | 79500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00476 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15899 |\n",
"| policy_loss | 4.02e-07 |\n",
"| value_loss | 7.96e-07 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 263 |\n",
"| iterations | 16000 |\n",
"| time_elapsed | 303 |\n",
"| total_timesteps | 80000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00489 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 15999 |\n",
"| policy_loss | 8.73e-07 |\n",
"| value_loss | 3.07e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 262 |\n",
"| iterations | 16100 |\n",
"| time_elapsed | 306 |\n",
"| total_timesteps | 80500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00329 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16099 |\n",
"| policy_loss | 5.7e-08 |\n",
"| value_loss | 2.39e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 262 |\n",
"| iterations | 16200 |\n",
"| time_elapsed | 308 |\n",
"| total_timesteps | 81000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.003 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16199 |\n",
"| policy_loss | -8.02e-08 |\n",
"| value_loss | 1.06e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 261 |\n",
"| iterations | 16300 |\n",
"| time_elapsed | 311 |\n",
"| total_timesteps | 81500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00325 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16299 |\n",
"| policy_loss | -3.32e-07 |\n",
"| value_loss | 1.15e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 261 |\n",
"| iterations | 16400 |\n",
"| time_elapsed | 313 |\n",
"| total_timesteps | 82000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00312 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16399 |\n",
"| policy_loss | -3.28e-07 |\n",
"| value_loss | 1.56e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 16500 |\n",
"| time_elapsed | 316 |\n",
"| total_timesteps | 82500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00269 |\n",
"| explained_variance | 0.000353 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16499 |\n",
"| policy_loss | -1.24e-08 |\n",
"| value_loss | 5.1e-09 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 16600 |\n",
"| time_elapsed | 318 |\n",
"| total_timesteps | 83000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00254 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16599 |\n",
"| policy_loss | -9.92e-08 |\n",
"| value_loss | 1.73e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 16700 |\n",
"| time_elapsed | 320 |\n",
"| total_timesteps | 83500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0025 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16699 |\n",
"| policy_loss | 3.89e-07 |\n",
"| value_loss | 2.8e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 16800 |\n",
"| time_elapsed | 322 |\n",
"| total_timesteps | 84000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0025 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16799 |\n",
"| policy_loss | 2.1e-07 |\n",
"| value_loss | 9.55e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 16900 |\n",
"| time_elapsed | 324 |\n",
"| total_timesteps | 84500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00249 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16899 |\n",
"| policy_loss | 1.48e-07 |\n",
"| value_loss | 3.85e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 17000 |\n",
"| time_elapsed | 326 |\n",
"| total_timesteps | 85000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00503 |\n",
"| explained_variance | -1.85 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 16999 |\n",
"| policy_loss | -3.69e-07 |\n",
"| value_loss | 8.96e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 260 |\n",
"| iterations | 17100 |\n",
"| time_elapsed | 328 |\n",
"| total_timesteps | 85500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0023 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17099 |\n",
"| policy_loss | -5.25e-09 |\n",
"| value_loss | 1.53e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17200 |\n",
"| time_elapsed | 330 |\n",
"| total_timesteps | 86000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00249 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17199 |\n",
"| policy_loss | 2.65e-07 |\n",
"| value_loss | 1.3e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17300 |\n",
"| time_elapsed | 332 |\n",
"| total_timesteps | 86500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00247 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17299 |\n",
"| policy_loss | 9.03e-08 |\n",
"| value_loss | 2.69e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17400 |\n",
"| time_elapsed | 334 |\n",
"| total_timesteps | 87000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00132 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17399 |\n",
"| policy_loss | 8.05e-08 |\n",
"| value_loss | 4.99e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17500 |\n",
"| time_elapsed | 337 |\n",
"| total_timesteps | 87500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00468 |\n",
"| explained_variance | -5.09 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17499 |\n",
"| policy_loss | -1.33e-07 |\n",
"| value_loss | 5.81e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17600 |\n",
"| time_elapsed | 339 |\n",
"| total_timesteps | 88000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00132 |\n",
"| explained_variance | -0.00905 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17599 |\n",
"| policy_loss | 2.64e-08 |\n",
"| value_loss | 5.46e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17700 |\n",
"| time_elapsed | 341 |\n",
"| total_timesteps | 88500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00153 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17699 |\n",
"| policy_loss | 2.11e-07 |\n",
"| value_loss | 2.43e-06 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17800 |\n",
"| time_elapsed | 342 |\n",
"| total_timesteps | 89000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00149 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17799 |\n",
"| policy_loss | -4.03e-07 |\n",
"| value_loss | 8.55e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 17900 |\n",
"| time_elapsed | 344 |\n",
"| total_timesteps | 89500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00149 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17899 |\n",
"| policy_loss | 5.53e-08 |\n",
"| value_loss | 3.97e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18000 |\n",
"| time_elapsed | 346 |\n",
"| total_timesteps | 90000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00146 |\n",
"| explained_variance | 0.000154 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 17999 |\n",
"| policy_loss | 5.49e-08 |\n",
"| value_loss | 1.86e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18100 |\n",
"| time_elapsed | 348 |\n",
"| total_timesteps | 90500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00143 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18099 |\n",
"| policy_loss | 1.66e-08 |\n",
"| value_loss | 2.37e-08 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18200 |\n",
"| time_elapsed | 350 |\n",
"| total_timesteps | 91000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00167 |\n",
"| explained_variance | 5.96e-08 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18199 |\n",
"| policy_loss | 9.68e-08 |\n",
"| value_loss | 5.07e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18300 |\n",
"| time_elapsed | 352 |\n",
"| total_timesteps | 91500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00174 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18299 |\n",
"| policy_loss | 7.17e-07 |\n",
"| value_loss | 2.34e-05 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18400 |\n",
"| time_elapsed | 353 |\n",
"| total_timesteps | 92000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00109 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18399 |\n",
"| policy_loss | 5.21e-08 |\n",
"| value_loss | 3.2e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18500 |\n",
"| time_elapsed | 356 |\n",
"| total_timesteps | 92500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00121 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18499 |\n",
"| policy_loss | -2.66e-08 |\n",
"| value_loss | 1.62e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18600 |\n",
"| time_elapsed | 358 |\n",
"| total_timesteps | 93000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00119 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18599 |\n",
"| policy_loss | -2.02e-08 |\n",
"| value_loss | 4.06e-08 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18700 |\n",
"| time_elapsed | 360 |\n",
"| total_timesteps | 93500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00121 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18699 |\n",
"| policy_loss | 1.87e-07 |\n",
"| value_loss | 3.02e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 18800 |\n",
"| time_elapsed | 362 |\n",
"| total_timesteps | 94000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00118 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18799 |\n",
"| policy_loss | 6.23e-08 |\n",
"| value_loss | 4.65e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 18900 |\n",
"| time_elapsed | 365 |\n",
"| total_timesteps | 94500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00108 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18899 |\n",
"| policy_loss | 2.17e-08 |\n",
"| value_loss | 5.12e-08 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19000 |\n",
"| time_elapsed | 367 |\n",
"| total_timesteps | 95000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0012 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 18999 |\n",
"| policy_loss | -2.02e-07 |\n",
"| value_loss | 4.04e-06 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19100 |\n",
"| time_elapsed | 369 |\n",
"| total_timesteps | 95500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00114 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19099 |\n",
"| policy_loss | 5.66e-08 |\n",
"| value_loss | 3.24e-07 |\n",
"------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19200 |\n",
"| time_elapsed | 371 |\n",
"| total_timesteps | 96000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00104 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19199 |\n",
"| policy_loss | -4.69e-08 |\n",
"| value_loss | 2.49e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19300 |\n",
"| time_elapsed | 373 |\n",
"| total_timesteps | 96500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00105 |\n",
"| explained_variance | -1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19299 |\n",
"| policy_loss | 1.12e-07 |\n",
"| value_loss | 2.48e-06 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19400 |\n",
"| time_elapsed | 375 |\n",
"| total_timesteps | 97000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.0239 |\n",
"| explained_variance | 0.186 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19399 |\n",
"| policy_loss | -2.59e-06 |\n",
"| value_loss | 2.3e-07 |\n",
"-------------------------------------\n",
"-------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19500 |\n",
"| time_elapsed | 376 |\n",
"| total_timesteps | 97500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00119 |\n",
"| explained_variance | 1.19e-07 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19499 |\n",
"| policy_loss | -5.15e-09 |\n",
"| value_loss | 3.86e-08 |\n",
"-------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19600 |\n",
"| time_elapsed | 378 |\n",
"| total_timesteps | 98000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00122 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19599 |\n",
"| policy_loss | 4.79e-08 |\n",
"| value_loss | 2.26e-07 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19700 |\n",
"| time_elapsed | 380 |\n",
"| total_timesteps | 98500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00124 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19699 |\n",
"| policy_loss | 2.39e-07 |\n",
"| value_loss | 4.65e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 19800 |\n",
"| time_elapsed | 382 |\n",
"| total_timesteps | 99000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00129 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19799 |\n",
"| policy_loss | -1.7e-07 |\n",
"| value_loss | 2.56e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 259 |\n",
"| iterations | 19900 |\n",
"| time_elapsed | 384 |\n",
"| total_timesteps | 99500 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00122 |\n",
"| explained_variance | -4.45 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19899 |\n",
"| policy_loss | 4.92e-08 |\n",
"| value_loss | 1.89e-06 |\n",
"------------------------------------\n",
"------------------------------------\n",
"| rollout/ | |\n",
"| ep_len_mean | 3.97e+04 |\n",
"| ep_rew_mean | -0.259 |\n",
"| time/ | |\n",
"| fps | 258 |\n",
"| iterations | 20000 |\n",
"| time_elapsed | 386 |\n",
"| total_timesteps | 100000 |\n",
"| train/ | |\n",
"| entropy_loss | -0.00103 |\n",
"| explained_variance | 0 |\n",
"| learning_rate | 0.0007 |\n",
"| n_updates | 19999 |\n",
"| policy_loss | 4.79e-08 |\n",
"| value_loss | 2.66e-07 |\n",
"------------------------------------\n",
"Saved A2C model to: G:\\My Drive\\Bots DRL\\DRL\\DRL-MT5-Lab\\notebooks\\models\\a2c_EURUSD_M15.zip\n"
]
}
],
"source": [
"# === Train A2C (same env/data as PPO) ===\n",
"import os, json\n",
"from pathlib import Path\n",
"from stable_baselines3 import A2C\n",
"from stable_baselines3.common.vec_env import DummyVecEnv\n",
"from stable_baselines3.common.monitor import Monitor\n",
"\n",
"def make_env():\n",
" return Monitor(TradingEnv(df_train, feature_cols))\n",
"\n",
"env_a2c = DummyVecEnv([make_env])\n",
"\n",
"a2c = A2C(\"MlpPolicy\", env_a2c,\n",
" verbose=1,\n",
" n_steps=5, # SB3 default\n",
" gamma=0.99,\n",
" learning_rate=7e-4)\n",
"\n",
"total_timesteps = int(os.getenv(\"TOTAL_TIMESTEPS\", \"100000\"))\n",
"a2c.learn(total_timesteps=total_timesteps)\n",
"\n",
"Path(\"models\").mkdir(exist_ok=True)\n",
"a2c_path = Path(\"models\") / f\"a2c_{SYMBOL}_{TIMEFRAME}.zip\"\n",
"a2c.save(a2c_path.as_posix())\n",
"\n",
"# (Write once is enough; safe to overwrite)\n",
"with open(Path(\"models\") / \"selected_features.json\", \"w\", encoding=\"utf-8\") as f:\n",
" json.dump(feature_cols, f, indent=2)\n",
"\n",
"print(\"Saved A2C model to:\", a2c_path.resolve())\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "92af869b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cpu device\n",
"Saved DQN model to: G:\\My Drive\\Bots DRL\\DRL\\DRL-MT5-Lab\\notebooks\\models\\dqn_EURUSD_M15.zip\n"
]
}
],
"source": [
"# === Train DQN (same env/data as PPO) ===\n",
"import os, json\n",
"from pathlib import Path\n",
"from stable_baselines3 import DQN\n",
"from stable_baselines3.common.vec_env import DummyVecEnv\n",
"from stable_baselines3.common.monitor import Monitor\n",
"\n",
"def make_env():\n",
" return Monitor(TradingEnv(df_train, feature_cols))\n",
"\n",
"env_dqn = DummyVecEnv([make_env])\n",
"\n",
"dqn = DQN(\"MlpPolicy\", env_dqn,\n",
" verbose=1,\n",
" learning_rate=1e-3,\n",
" buffer_size=100_000,\n",
" learning_starts=1_000,\n",
" batch_size=64,\n",
" tau=1.0,\n",
" gamma=0.99,\n",
" train_freq=4,\n",
" target_update_interval=1_000,\n",
" exploration_fraction=0.1,\n",
" exploration_final_eps=0.01)\n",
"\n",
"total_timesteps = int(os.getenv(\"TOTAL_TIMESTEPS\", \"100000\"))\n",
"dqn.learn(total_timesteps=total_timesteps)\n",
"\n",
"Path(\"models\").mkdir(exist_ok=True)\n",
"dqn_path = Path(\"models\") / f\"dqn_{SYMBOL}_{TIMEFRAME}.zip\"\n",
"dqn.save(dqn_path.as_posix())\n",
"\n",
"# (Write once is enough; safe to overwrite)\n",
"with open(Path(\"models\") / \"selected_features.json\", \"w\", encoding=\"utf-8\") as f:\n",
" json.dump(feature_cols, f, indent=2)\n",
"\n",
"print(\"Saved DQN model to:\", dqn_path.resolve())\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "drl",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.18"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink