gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: run benchmark on gpt-4o & llama 3.1 (#497)

Browse Source 
* Run benchmark on gpt-4o & llama 3.1

* update link
This commit is contained in:
you-n-g
2024-11-26 12:02:09 +08:00
committed by GitHub
parent ac18f6f892
commit 1acb1dab8f
6 changed files with 57 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/eva_utils.py
+2 -6
View File
@@ -221,15 +221,11 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
str(resp_dict["output_format_feedback"]),
resp_dict["output_format_decision"],
)
except json.JSONDecodeError as e:
raise ValueError("Failed to decode JSON response from API.") from e
except KeyError as e:
except (KeyError, json.JSONDecodeError) as e:
attempts += 1
if attempts >= max_attempts:
raise KeyError(
"Response from API is missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
"Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
) from e
return "Failed to evaluate output format after multiple attempts.", False
rdagent/components/coder/factor_coder/evolving_strategy.py
+14 -8
View File
@@ -158,14 +158,20 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[:-1]
elif len(queried_similar_error_knowledge_to_render) > 0:
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
code = json.loads(
APIBackend(
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
).build_messages_and_create_chat_completion(
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
)
)["code"]
return code
for _ in range(10):
try:
code = json.loads(
APIBackend(
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
).build_messages_and_create_chat_completion(
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
)
)["code"]
return code
except json.decoder.JSONDecodeError:
pass
else:
return "" # return empty code if failed to get code after 10 attempts
def assign_code_list_to_evo(self, code_list, evo):
for index in range(len(evo.sub_tasks)):