2021-12-22 12:04:38 +01:00
|
|
|
import pandas as pd
|
2022-01-26 23:22:43 +01:00
|
|
|
from config.types import RawConfig
|
2021-12-22 12:04:38 +01:00
|
|
|
from typing import Optional
|
2021-12-26 12:15:11 +01:00
|
|
|
from utils.helpers import weighted_average
|
2022-01-29 06:41:40 +01:00
|
|
|
from training.types import Stats
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
def launch_wandb(project_name:str, default_config: RawConfig, sweep:bool=False) -> Optional[object]:
|
2021-12-22 12:04:38 +01:00
|
|
|
from wandb_setup import get_wandb
|
|
|
|
|
wandb = get_wandb()
|
2022-01-07 15:32:18 +01:00
|
|
|
if wandb is None:
|
|
|
|
|
raise Exception("Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)")
|
|
|
|
|
|
2021-12-22 12:04:38 +01:00
|
|
|
elif sweep:
|
2022-01-23 18:37:43 +01:00
|
|
|
wandb.init(project=project_name, config = vars(default_config))
|
2021-12-22 12:04:38 +01:00
|
|
|
return wandb
|
|
|
|
|
else:
|
2022-01-23 18:37:43 +01:00
|
|
|
wandb.init(project=project_name, config = vars(default_config), reinit=True)
|
2021-12-22 12:04:38 +01:00
|
|
|
return wandb
|
|
|
|
|
|
|
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
def override_config_with_wandb_values(wandb: Optional[object], raw_config: RawConfig) -> RawConfig:
|
|
|
|
|
if wandb is None: return raw_config
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
wandb_config: dict = wandb.config
|
2021-12-27 21:59:22 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
config_dict = vars(raw_config)
|
|
|
|
|
for k in config_dict:
|
|
|
|
|
config_dict[k] = wandb_config[k]
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
return RawConfig(**config_dict)
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-29 06:41:40 +01:00
|
|
|
def send_report_to_wandb(stats: Stats, wandb:Optional[object]):
|
2021-12-22 12:04:38 +01:00
|
|
|
if wandb is None: return
|
|
|
|
|
|
2022-01-10 14:17:06 +01:00
|
|
|
run = wandb.run
|
|
|
|
|
run.save()
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-29 06:41:40 +01:00
|
|
|
for key, value in stats.items():
|
2022-01-10 14:17:06 +01:00
|
|
|
run.log({ key: value })
|
2021-12-22 12:04:38 +01:00
|
|
|
|
|
|
|
|
run.finish()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|