2021-12-22 12:04:38 +01:00
|
|
|
import pandas as pd
|
2022-01-26 23:22:43 +01:00
|
|
|
from config.types import RawConfig
|
2021-12-22 12:04:38 +01:00
|
|
|
from typing import Optional
|
2021-12-26 12:15:11 +01:00
|
|
|
from utils.helpers import weighted_average
|
2022-01-29 06:41:40 +01:00
|
|
|
from training.types import Stats
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-02-17 19:22:17 +01:00
|
|
|
|
|
|
|
|
def launch_wandb(
|
|
|
|
|
project_name: str, default_config: RawConfig, sweep: bool = False
|
|
|
|
|
) -> Optional[object]:
|
2021-12-22 12:04:38 +01:00
|
|
|
from wandb_setup import get_wandb
|
2022-02-17 19:22:17 +01:00
|
|
|
|
|
|
|
|
wandb = get_wandb()
|
2022-01-07 15:32:18 +01:00
|
|
|
if wandb is None:
|
2022-02-17 19:22:17 +01:00
|
|
|
raise Exception(
|
|
|
|
|
"Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)"
|
|
|
|
|
)
|
2022-01-07 15:32:18 +01:00
|
|
|
|
2021-12-22 12:04:38 +01:00
|
|
|
elif sweep:
|
2022-02-17 19:22:17 +01:00
|
|
|
wandb.init(project=project_name, config=vars(default_config))
|
2021-12-22 12:04:38 +01:00
|
|
|
return wandb
|
|
|
|
|
else:
|
2022-02-17 19:22:17 +01:00
|
|
|
wandb.init(project=project_name, config=vars(default_config), reinit=True)
|
2021-12-22 12:04:38 +01:00
|
|
|
return wandb
|
2022-02-17 19:22:17 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def override_config_with_wandb_values(
|
|
|
|
|
wandb: Optional[object], raw_config: RawConfig
|
|
|
|
|
) -> RawConfig:
|
|
|
|
|
if wandb is None:
|
|
|
|
|
return raw_config
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
wandb_config: dict = wandb.config
|
2022-02-17 19:22:17 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
config_dict = vars(raw_config)
|
|
|
|
|
for k in config_dict:
|
|
|
|
|
config_dict[k] = wandb_config[k]
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-23 18:37:43 +01:00
|
|
|
return RawConfig(**config_dict)
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-02-17 19:22:17 +01:00
|
|
|
|
|
|
|
|
def send_report_to_wandb(stats: Stats, wandb: Optional[object]):
|
|
|
|
|
if wandb is None:
|
|
|
|
|
return
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-10 14:17:06 +01:00
|
|
|
run = wandb.run
|
|
|
|
|
run.save()
|
2021-12-22 12:04:38 +01:00
|
|
|
|
2022-01-29 06:41:40 +01:00
|
|
|
for key, value in stats.items():
|
2022-02-17 19:22:17 +01:00
|
|
|
run.log({key: value})
|
2021-12-22 12:04:38 +01:00
|
|
|
|
|
|
|
|
run.finish()
|