feat: joblib cache (#749)

* cache function

* fix test

* bin cache

* fix test

* fix test

* fix test

* cache for different source

* cache for localenv

* remove unnecessary log

* reformat

* remove unrelated modify
This commit is contained in:
Tim
2025-04-04 12:08:18 +08:00
committed by GitHub
parent 7f82894fc2
commit 43c8ed2aaf
7 changed files with 65 additions and 8 deletions
@@ -35,6 +35,17 @@ print(f"test_ids length: {len(test_ids)}")
train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.8, random_state=42)
import sys
import reprlib
from joblib.memory import MemorizedFunc
def get_original_code(func):
if isinstance(func, MemorizedFunc):
return func.func.__code__
return func.__code__
print("train_X:", aRepr.repr(train_X))
print("train_y:", aRepr.repr(train_y))
print("val_X:", aRepr.repr(val_X))
@@ -46,10 +57,12 @@ print(f"val_X.shape: {val_X.shape}" if hasattr(val_X, 'shape') else f"val_X leng
print(f"val_y.shape: {val_y.shape}" if hasattr(val_y, 'shape') else f"val_y length: {len(val_y)}")
def debug_info_print(func):
def wrapper(*args, **kwargs):
original_code = get_original_code(func)
def local_trace(frame, event, arg):
if event == "return" and frame.f_code == func.__code__:
if event == "return" and frame.f_code == original_code:
print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20)
for k, v in frame.f_locals.items():
printed = aRepr.repr(v)
@@ -40,6 +40,11 @@ model_coder:
{{ feature_code }}
2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
3. If the model can both be implemented by PyTorch and Tensorflow, please use pytorch for broader compatibility.
4. You should use the following cache decorator to cache the results of the function:
```python
from joblib import Memory
memory = Memory(location='/tmp/cache', verbose=0)
@memory.cache``
## Output Format
{% if out_spec %}