feat: add type checker to api backend & align litellm and old backend (#647)

* move cache auto continue and retry to all api backend

* add type checker to json mode output

* fix CI

* feat: Add json_mode handling and streaming support in chat completion function

* lint

* fix a bug when returning a dict which value could contain int or bool

* remove litellm

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>
Co-authored-by: Young <afe.young@gmail.com>

rdagent/components/coder/model_coder/eva_utils.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Tuple
+from typing import Dict, Tuple
 
 import numpy as np
 from jinja2 import Environment, StrictUndefined
@@ -177,6 +177,7 @@ class ModelFinalEvaluator(CoSTEEREvaluator):
                 user_prompt=user_prompt,
                 system_prompt=system_prompt,
                 json_mode=True,
+                json_target_type=Dict[str, str | bool | int],
             ),
         )
         if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[

rdagent/components/coder/model_coder/evolving_strategy.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from typing import Dict
 
 from jinja2 import Environment, StrictUndefined
 
@@ -96,6 +97,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
                 user_prompt=user_prompt,
                 system_prompt=system_prompt,
                 json_mode=True,
+                json_target_type=Dict[str, str],
             ),
         )["code"]
         return code