gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-01 17:37:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support more models for kaggle scenario (#223)

Browse Source 
* init commit for XGBoost

* fix some bugs

* CI issues

* CI issues

* CI issue

* edit prompts for kaggle scenario & fix some bugs

* Revised Prompts To Improve Performance on Model Type & Support of Random Forest

* edit prompts & modify evaluator.py to adapt to Kaggle scenario

* edit prompts

* fix some bugs

* CI issues

---------

Co-authored-by: Taozhi Wang <taozhi.mark.wang@gmail.com>
Co-authored-by: Xisen-Wang <xisen_application@163.com>
This commit is contained in:
TPLin22
2024-08-23 15:13:50 +08:00
committed by GitHub
parent c8e8989edc
commit 9a792cc21e
8 changed files with 226 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/model_coder/CoSTEER/evaluators.py
+17 -4
View File
@@ -26,13 +26,19 @@ evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yam
def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]:
if target_shape is None or prediction is None:
return "No output generated from the model. No shape evaluation conducted.", False
return (
"No output generated from the model. No shape evaluation conducted.",
False,
)
pre_shape = prediction.shape
if pre_shape == target_shape:
return "The shape of the output is correct.", True
else:
return f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.", False
return (
f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.",
False,
)
def reshape_tensor(original_tensor, target_shape):
@@ -50,7 +56,10 @@ def value_evaluator(
if prediction is None:
return "No output generated from the model. Skip value evaluation", False
elif target is None:
return "No ground truth output provided. Value evaluation not impractical", False
return (
"No ground truth output provided. Value evaluation not impractical",
False,
)
else:
# Calculate the mean absolute difference
diff = torch.mean(torch.abs(target - prediction)).item()
@@ -270,7 +279,11 @@ class ModelCoderEvaluator(Evaluator):
else:
gt_tensor = None
shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
if target_task.model_type == "XGBoost":
shape_feedback = "Not applicable for XGBoost models"
shape_decision = True
else:
shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor)
code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
target_task=target_task,
rdagent/components/coder/model_coder/model.py
+33 -16
View File
@@ -6,7 +6,9 @@ import uuid
from pathlib import Path
from typing import Any, Dict, Optional
import numpy as np
import torch
import xgboost as xgb
from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
from rdagent.core.exception import CodeFormatError
@@ -32,9 +34,7 @@ class ModelTask(Task):
self.architecture: str = architecture
self.variables: str = variables
self.hyperparameters: str = hyperparameters
self.model_type: str = (
model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model
)
self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
def get_task_information(self):
return f"""name: {self.name}
@@ -95,9 +95,17 @@ class ModelFBWorkspace(FBWorkspace):
if cache_file_path.exists():
return pickle.load(open(cache_file_path, "rb"))
mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
model_cls = mod.model_cls
if self.target_task.model_type == "Tabular":
if self.target_task.model_type != "XGBoost":
model_cls = mod.model_cls
if self.target_task.model_type == "XGBoost":
X_simulated = np.random.rand(100, num_features) # 100 samples, `num_features` features each
y_simulated = np.random.randint(0, 2, 100) # Binary target for example
params = mod.get_params()
num_round = mod.get_num_round()
dtrain = xgb.DMatrix(X_simulated, label=y_simulated)
elif self.target_task.model_type == "Tabular":
input_shape = (batch_size, num_features)
m = model_cls(num_features=input_shape[1])
data = torch.full(input_shape, input_value)
@@ -113,21 +121,30 @@ class ModelFBWorkspace(FBWorkspace):
else:
raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
# Initialize all parameters of `m` to `param_init_value`
for _, param in m.named_parameters():
param.data.fill_(param_init_value)
# Execute the model
if self.target_task.model_type == "Graph":
out = m(*data)
if self.target_task.model_type == "XGBoost":
bst = xgb.train(params, dtrain, num_round)
y_pred = bst.predict(dtrain)
execution_model_output = y_pred
execution_feedback_str = "Execution successful, model trained and predictions made."
else:
out = m(data)
# Initialize all parameters of `m` to `param_init_value`
for _, param in m.named_parameters():
param.data.fill_(param_init_value)
execution_model_output = out.cpu().detach()
execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
# Execute the model
if self.target_task.model_type == "Graph":
out = m(*data)
else:
out = m(data)
execution_model_output = out.cpu().detach()
execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
if MODEL_IMPL_SETTINGS.enable_execution_cache:
pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb"))
pickle.dump(
(execution_feedback_str, execution_model_output),
open(cache_file_path, "wb"),
)
except Exception as e:
execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
rdagent/components/coder/model_coder/prompts.yaml
+3 -3
View File
@@ -15,7 +15,7 @@ extract_model_formulation_system: |-
"hyperparameter_name_2": "value of hyperparameter 2",
"hyperparameter_name_3": "value of hyperparameter 3"
},
"model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph"
"model_type": "Tabular or TimeSeries or Graph or XGBoost" # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost"
}
}
Eg.
@@ -34,7 +34,7 @@ extract_model_formulation_system: |-
"hyperparameter_name_2": "value of hyperparameter 2",
"hyperparameter_name_3": "value of hyperparameter 3"
},
"model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph"
"model_type": "Tabular or TimeSeries or Graph or RandomForest or XGBoost" # If torch & Neural network models are required, the choice should be one of "Tabular", "TimeSeries", or "Graph"
}
}
such format content should be begin with ```json and end with ``` and the content should be in json format.
@@ -99,7 +99,7 @@ evaluator_code_feedback:
The user will provide the source python code and the execution error message if execution failed.
The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
User has also compared the output generated by the user's code and the ground truth code. The user will provide you some analyze result comparing two output. You may find some error in the code which caused the difference between the two output.
User has also compared the output generated by the user's code and the ground truth code. The user will provide you some analysis results comparing two output. You may find some error in the code which caused the difference between the two output.
If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct to the description and to the scenario.