From 7bb2b9c38230a925b431703c5659e614c2b35749 Mon Sep 17 00:00:00 2001 From: hlillemark Date: Sat, 15 Mar 2025 22:46:46 -0700 Subject: [PATCH 1/2] Change analyse results to take all possible success signals --- analyse_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyse_results.py b/analyse_results.py index 552865e..c83a83a 100644 --- a/analyse_results.py +++ b/analyse_results.py @@ -76,7 +76,7 @@ def analyze_json_file(file_path): if 'turns' in data and isinstance(data['turns'], list): for turn in reversed(data['turns']): # Check turns from the end if turn.get('role') == 'system' and isinstance(turn.get('content'), str): - if "Task successful ended with code : 2" in turn['content'] or "Task ended in score: 1" in turn["content"]: + if "Task successful ended with code : 2" in turn['content'] or "Task ended with score : 1" in turn["content"] or "Task ended in score: 1" in turn["content"]: return True return False except FileNotFoundError: From 5a230e707e0f1345dfa95967993fe9857c3b295d Mon Sep 17 00:00:00 2001 From: hlillemark Date: Mon, 17 Mar 2025 11:53:31 -0700 Subject: [PATCH 2/2] Edit eval script to work with ollama model --- evaluation_script.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/evaluation_script.py b/evaluation_script.py index 2f247d8..e9f6025 100644 --- a/evaluation_script.py +++ b/evaluation_script.py @@ -446,6 +446,12 @@ def make_profiles(agent_names, models, apis, template_profile="profiles/collab_p "model": models[index], "url": url } + elif apis[index] == "ollama": + profile["model"] = { + "api": "ollama", + "model": models[index], + "embedding": "ollama" + } else: profile["model"] = models[index]