feat: joblib cache (#749)

* cache function

* fix test

* bin cache

* fix test

* fix test

* fix test

* cache for different source

* cache for localenv

* remove unnecessary log

* reformat

* remove unrelated modify
This commit is contained in:
Tim
2025-04-04 12:08:18 +08:00
committed by GitHub
parent 7f82894fc2
commit 43c8ed2aaf
7 changed files with 65 additions and 8 deletions
@@ -35,6 +35,17 @@ print(f"test_ids length: {len(test_ids)}")
train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.8, random_state=42)
import sys
import reprlib
from joblib.memory import MemorizedFunc
def get_original_code(func):
if isinstance(func, MemorizedFunc):
return func.func.__code__
return func.__code__
print("train_X:", aRepr.repr(train_X))
print("train_y:", aRepr.repr(train_y))
print("val_X:", aRepr.repr(val_X))
@@ -46,10 +57,12 @@ print(f"val_X.shape: {val_X.shape}" if hasattr(val_X, 'shape') else f"val_X leng
print(f"val_y.shape: {val_y.shape}" if hasattr(val_y, 'shape') else f"val_y length: {len(val_y)}")
def debug_info_print(func):
def wrapper(*args, **kwargs):
original_code = get_original_code(func)
def local_trace(frame, event, arg):
if event == "return" and frame.f_code == func.__code__:
if event == "return" and frame.f_code == original_code:
print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20)
for k, v in frame.f_locals.items():
printed = aRepr.repr(v)