import argparse import json import shutil import subprocess import time from datetime import datetime import re import sys import os import time import filecmp import json import glob import socket from tqdm import tqdm import boto3 # Calculate project root directory project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Define tasks directory tasks_dir = os.path.dirname(os.path.abspath(__file__)) BLOCKED_ACTIONS_COOKING = [ '!activate', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endGoal', '!entities', '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', '!goToBed', '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', '!restart', '!setMode', '!stay', '!stfu', '!stop' ] BLOCKED_ACTIONS_CRAFTING = [ '!activate', '!attack', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endConversation', '!endGoal', '!entities', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', '!goToBed', '!help', '!modes', '!newAction', '!putInChest', '!restart', '!searchForEntity', '!setMode', '!stay', '!stfu', '!stop', '!takeFromChest', '!viewChest' ] BLOCKED_ACTIONS_CONSTRUCTION = [ '!activate', '!attackPlayer', '!clearChat', '!clearFurnace', '!collectBlocks', '!consume', '!craftable', '!discard', '!endConversation', '!endGoal', '!entities', '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', '!goToBed', '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', '!restart', '!searchForBlock', '!searchForEntity', '!setMode', '!stay', '!stfu', '!stop', '!takeFromChest', '!viewChest', '!craftRecipe', '!smeltItem' ] def analyze_json_file(file_path): """ Analyzes a single JSON file to extract the task outcome. Args: file_path (str): Path to the JSON file. Returns: str or None: The task outcome string if found, otherwise None. """ try: with open(file_path, 'r') as f: data = json.load(f) if "turns" in data: for turn in data["turns"]: if turn.get("role") == "system" and "content" in turn: if isinstance(turn["content"], str) and "Task ended with score : " in turn["content"]: if "Task ended with score : 1" in turn["content"]: return 1 elif "Task ended with score : 0" in turn["content"]: return 0 else: score = float(turn["content"].split(":")[-1].strip()) return score return None except FileNotFoundError: print(f"Error: File not found: {file_path}") return None except json.JSONDecodeError: print(f"Error: Invalid JSON format in: {file_path}") return None except Exception as e: print(f"An unexpected error occurred while processing {file_path}: {e}") return None def extract_result(folder_path): folder_name = os.path.basename(folder_path) json_files = glob.glob(os.path.join(folder_path, "*.json")) # assert len(json_files) == 2, f"Expected 2 json files in {folder_name}, found {len(json_files)}" if not json_files: return None else: score = None curr_score = 0 for json_file in json_files: score = analyze_json_file(json_file) if score is not None: max_score = max(score, curr_score) curr_score = max_score return curr_score def aggregate_results(local_folders): """ Aggregates the analysis results for each folder. Args: local_folders (list): List of local folder paths containing the JSON files. Returns: dict: A dictionary where keys are folder names and values are the aggregated outcomes. """ aggregated_data = {} total = 0 successful = 0 successful_tasks = [] task_type = local_folders[0].split("/")[-2] if "cooking" in task_type: task_type = "cooking" elif "techtree" in task_type: task_type = "techtree" elif "construction" in task_type: task_type = "construction" for folder_path in tqdm(local_folders): folder_name = os.path.basename(folder_path) try: result = extract_result(folder_path) if result == 1: successful_tasks.append(folder_name) if result is not None: total += 1 successful += result except Exception as e: print(f"Error processing {folder_name}: {e}") successful_tasks.sort() if task_type == "construction": successful = successful / total return { "total": total, "successful": successful, } def check_folder_results(folder_path): """ Evaluate all JSON files in a folder and its subfolders and calculate success metrics. Args: folder_path (str): Path to the folder containing JSON log files. Returns: dict: A dictionary with success metrics. """ print(f"Checking results in folder: {folder_path}") # Check if the folder exists if not os.path.exists(folder_path): print(f"Error: Folder not found: {folder_path}") return None # Find all subfolders (task IDs) in the given folder if os.path.isdir(folder_path): subfolders = [f for f in glob.glob(os.path.join(folder_path, "*")) if os.path.isdir(f)] if subfolders: # If there are subfolders, evaluate each subfolder print(f"Found {len(subfolders)} subfolders to evaluate") results = aggregate_results(subfolders) else: # If no subfolders, treat the folder itself as a results folder print("No subfolders found, evaluating the folder itself") results = aggregate_results([folder_path]) # Calculate success rate if results["total"] > 0: results["success_rate"] = results["successful"] / results["total"] else: results["success_rate"] = 0.0 # Print summary print("\n=== Evaluation Results ===") print(f"Total tasks evaluated: {results['total']}") if "construction" not in folder_path: print(f"Successful tasks: {results['successful']}") if "construction" not in folder_path: print(f"Success rate: {results['success_rate']:.2f}") else: print(f"Success rate: {results['successful']:.2f}") return results else: print(f"Error: {folder_path} is not a directory") return None def read_settings(file_path): """Read and parse the settings.js file to get agent profiles.""" # Ensure file_path is absolute or relative to project_root if not os.path.isabs(file_path): file_path = os.path.join(project_root, file_path) with open(file_path, 'r', encoding='utf-8') as file: content = file.read() # Remove `export default` and trailing commas content = re.sub(r'export\s+default', '', content) content = re.sub(r',\s*(?=[}\]])', '', content) # Remove JavaScript comments content = re.sub(r'//.*', '', content) # Remove trailing commas (e.g., before } or ]) content = re.sub(r',\s*(?=[}\]])', '', content) # Strip leading and trailing whitespace content = content.strip() json_data = json.loads(content) profiles = json_data['profiles'] ## profiles is a list of strings like "./andy.json" and "./bob.json" agent_names = [profile.split('/')[-1].split('.')[0] for profile in profiles] return agent_names def update_keys_json(): """Update the keys.json file with the specified key-value pair.""" keys_example_path = os.path.join(project_root, "keys.example.json") keys_path = os.path.join(project_root, "keys.json") with open(keys_example_path, 'r', encoding='utf-8') as file: content = file.read() data = json.loads(content) # Update keys with environment variables for key in data.keys(): env_value = os.getenv(key) # Fetch from environment variables if env_value: # If the variable exists, update it data[key] = env_value with open(keys_path, 'w', encoding='utf-8') as file: json.dump(data, file, indent=4) def set_environment_variable_tmux_session(session_name, key, value): """Set an environment variable for the current process.""" subprocess.run(["tmux", "send-keys", "-t", session_name, f"export {key}={value}", "C-m"]) def launch_parallel_experiments(task_path, num_exp, exp_name, num_agents=2, model="gpt-4o-mini", api="openai", num_parallel=1, s3=False, bucket_name="mindcraft-experiments", template_profile="profiles/tasks/collab_profile.json", insecure_coding=False, url="http://127.0.0.1:8000/v1", max_messages=15, num_examples=2, no_pruning=False, block_conversation=False, run_in_tmux=True): # Resolve relative template_profile path if not os.path.isabs(template_profile): template_profile = os.path.join(project_root, template_profile) # Resolve relative task_path path if not os.path.isabs(task_path): task_path = os.path.join(project_root, task_path) with open(task_path, 'r', encoding='utf-8') as file: content = file.read() json_data = json.loads(content) task_ids = json_data.keys() task_type = json_data[list(task_ids)[0]]["type"] # split the task_ids into num_parallel groups task_ids = list(task_ids) task_ids_split = [task_ids[i::num_parallel] for i in range(num_parallel)] if task_type == "cooking": world_name = "Superflat" elif task_type == "techtree": world_name = "Forest" elif task_type == "construction": world_name = "Superflat" if run_in_tmux: servers = create_server_files("./server_data/", num_parallel, world_name=world_name) else: servers = [(f"./server_data_{i}/", 55916 + i) for i in range(num_parallel)] date_time = datetime.now().strftime("%m-%d_%H-%M") experiments_folder = f"experiments/{exp_name}_{date_time}" exp_name = f"{exp_name}_{date_time}" split_task_path = task_path.split("/") if len(split_task_path) > 1: task_path_name = split_task_path[-2] else: task_path_name = "tasks" s3_path = f"{bucket_name}/{task_type}/{model}/{task_path_name}/{exp_name}" # start wandb os.makedirs(experiments_folder, exist_ok=True) for i, server in enumerate(servers): launch_server_experiment(task_path, task_ids_split[i], num_exp, server, experiments_folder, exp_name, s3=s3, bucket_name=bucket_name, template_profile=template_profile, model=model, api=api, insecure_coding=insecure_coding, num_agents=num_agents, url=url, task_type=task_type, s3_path=s3_path, max_messages=max_messages, num_examples=num_examples, no_pruning=no_pruning, block_conversation=block_conversation, run_in_tmux=run_in_tmux) time.sleep(5) total_num_tasks = len(task_ids) total_num_experiments = total_num_tasks * num_exp total_run = 0 while total_run < total_num_experiments: results = aggregate_results([f"{experiments_folder}/{task_id}" for task_id in task_ids]) total_run = results["total"] print(f"Total tasks run: {total_run}/{total_num_experiments}") print(results) results["exp_name"] = exp_name results["template_profile"] = template_profile results["model"] = model results["api"] = api results["num_agents"] = num_agents results["task_path"] = task_path results["task_type"] = task_type results["max_messages"] = max_messages results["num_examples"] = num_examples with open(f"{experiments_folder}/results.txt", "w") as file: file.write(str(results)) if s3: cmd = f"aws s3 cp {experiments_folder}/results.txt s3://{s3_path}/results.txt" print(cmd) subprocess.run(cmd.split()) time.sleep(60) def launch_server_experiment(task_path, task_ids, num_exp, server, experiments_folder, exp_name="exp", num_agents=2, model="gpt-4o", api="openai", s3=False, bucket_name="mindcraft-experiments", template_profile="profiles/tasks/collab_profile.json", insecure_coding=False, url="http://127.0.0.1:8000/v1", task_type="techtree", s3_path="", max_messages=15, num_examples=2, no_pruning=False, block_conversation=False, run_in_tmux=True): # Resolve relative template_profile path if not os.path.isabs(template_profile): template_profile = os.path.join(project_root, template_profile) # Resolve relative task_path path if not os.path.isabs(task_path): task_path = os.path.join(project_root, task_path) experiments_folder = os.path.join(project_root, experiments_folder) server_path, server_port = server edit_file(os.path.join(server_path, "server.properties"), {"server-port": server_port}) mindserver_port = server_port - 55916 + 8080 # set up server and agents session_name = str(server_port - 55916) if num_agents == 1: agent_names = [f"Andy_{session_name}"] models = [model] apis = [api] elif num_agents == 2: agent_names = [f"Andy_{session_name}", f"Jill_{session_name}"] models = [model] * 2 apis = [api] * 2 else: # Lets use an ordered list of 10 human names. human_names = ["Andy", "Jill", "Bob", "Sally", "Mike", "Laura", "John", "Emma", "Tom", "Kate"] agent_names = [] for i in range(num_agents): name = human_names[i % len(human_names)] agent_names.append(f"{name}_{session_name}") models = [model] * num_agents apis = [api] * num_agents make_profiles(agent_names, models, apis, template_profile=template_profile, url=url) agent_profiles = [f"./{agent}.json" for agent in agent_names] if num_agents == 1: agent_profiles_str = f"'[\"{agent_profiles[0]}\"]'" elif num_agents == 2: agent_profiles_str = f"'[\"{agent_profiles[0]}\", \"{agent_profiles[1]}\"]'" else: agent_profiles_str = "'[" for agent in agent_profiles[:-1]: agent_profiles_str += f'\"{agent}\", ' agent_profiles_str += f"\"{agent_profiles[-1]}\"]'" print(agent_profiles_str) if run_in_tmux: print("run in tmux is true") launch_world(server_path, session_name="server_" + session_name, agent_names=agent_names, port=server_port) subprocess.run(['tmux', 'new-session', '-d', '-s', session_name], check=True) # set environment variables if run_in_tmux: set_environment_variable_tmux_session(session_name, "MINECRAFT_PORT", server_port) set_environment_variable_tmux_session(session_name, "MINDSERVER_PORT", mindserver_port) set_environment_variable_tmux_session(session_name, "PROFILES", agent_profiles_str) set_environment_variable_tmux_session(session_name, "MAX_MESSAGES", str(max_messages)) set_environment_variable_tmux_session(session_name, "NUM_EXAMPLES", str(num_examples)) set_environment_variable_tmux_session(session_name, "LOG_ALL", "true") if insecure_coding: set_environment_variable_tmux_session(session_name, "INSECURE_CODING", "true") make_ops(agent_names, session_name) else: agent_profiles_str = "[" for agent in agent_profiles[:-1]: agent_profiles_str += f"\"{agent}\", " agent_profiles_str += f"\"{agent_profiles[-1]}\"]" # print(agent_profiles_str) os.environ["PROFILES"] = agent_profiles_str os.environ["MAX_MESSAGES"] = str(max_messages) os.environ["NUM_EXAMPLES"] = str(num_examples) os.environ["LOG_ALL"] = "true" run_script(task_path, task_ids, num_exp, experiments_folder, agent_names, server_path, s3=s3, s3_path=s3_path, session_name=session_name, run_in_tmux=run_in_tmux) # add the bots as op # op_script_content = "sleep 5\n\op @p" * 20 # op_script_file = f"./tmp/op_script_{session_name}.sh" # make_script_file_and_run(op_script_content, "server_" + session_name, op_script_file) # blocked_actions = [] # if not no_pruning: # if task_type == "cooking": # blocked_actions = BLOCKED_ACTIONS_COOKING # elif task_type == "techtree": # blocked_actions = BLOCKED_ACTIONS_CRAFTING # elif task_type == "construction": # blocked_actions = BLOCKED_ACTIONS_CONSTRUCTION # if block_conversation: # blocked_actions += ["!endConversation", "!startConversation"] # set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", blocked_actions) # script_content = "" # for task_id in task_ids: # # Create a separate folder for each task_id # task_folder = os.path.join(experiments_folder, str(task_id)) # os.makedirs(task_folder, exist_ok=True) # assert os.path.exists(task_folder), f"Directory {task_folder} was not created" # print(f"Created directory: {task_folder}") # cmd = f"node main.js --task_path \'{task_path}\' --task_id {task_id}" # cp_cmd = f"cp {agent_names[0]}.json {server_path}bots/{agent_names[0]}/profile.json" # for _ in range(num_exp): # script_content += f"{cmd}\n" # script_content += "sleep 2\n" # for agent in agent_names: # agent_file_path = os.path.join(task_folder, f"{agent}_{_}.json") # script_content += f"echo 'Saving to {agent_file_path}'\n" # cp_cmd = f"cp bots/{agent}/memory.json {agent_file_path}" # script_content += f"echo '{cp_cmd}'\n" # script_content += f"{cp_cmd}\n" # script_content += "sleep 1\n" # if s3: # s3_cmd = f"aws s3 cp {agent_file_path} s3://{s3_path}/{task_id}/{agent}_{_}.json" # script_content += f"echo 'Uploading {agent_file_path} to S3'\n" # script_content += f"echo '{s3_cmd}'\n" # script_content += f"{s3_cmd}\n" # script_content += "sleep 1\n" # script_content += f"sleep 10\n" # if s3: # for agent in agent_names: # script_content += f"aws s3 cp bots/{agent} s3://{s3_path}/bots/{agent} --recursive\n" # # Create a temporary shell script file # script_file = f"./tmp/experiment_script_{session_name}.sh" # make_script_file_and_run(script_content, script_file, session_name=session_name, run_in_tmux=True) def run_script(task_path, task_ids, num_exp, experiments_folder, agent_names, server_path, s3=False, s3_path="mindcraft-experiments", session_name="0", run_in_tmux=True,): # Resolve relative task_path path if not os.path.isabs(task_path): task_path = os.path.join(project_root, task_path) # Resolve relative experiments_folder path if not os.path.isabs(experiments_folder): experiments_folder = os.path.join(project_root, experiments_folder) # Resolve relative server_path path if not os.path.isabs(server_path): server_path = os.path.join(project_root, server_path) # Construct command (assuming main.js is in root) main_js_path = os.path.join(project_root, "main.js") for exp in range(num_exp): for task_id in task_ids: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") exp_folder = os.path.join(experiments_folder, f"{task_id}_{exp}_{timestamp}") # Need to create the folder first if using subprocess and cwd os.makedirs(exp_folder, exist_ok=True) cmd = [ "node", main_js_path, "--task_path", task_path, "--task_id", task_id, "--agent_name", agent_names[0], "--agent_name", agent_names[1], "--server", server_path, "--logs_path", exp_folder, # Ensure logs_path is absolute or handled by main.js relative to root ] if s3: cmd.extend(["--s3", "--s3_path", s3_path]) script_content = " ".join(cmd) make_script_file_and_run(script_content, file_name=f"exp_{exp}_{task_id}_{timestamp}.sh", session_name=session_name, run_in_tmux=run_in_tmux) print(f"Launched Experiment {exp+1}/{num_exp} for Task {task_id}") time.sleep(1) # Stagger launches def make_ops(agent_names, session_name): """Make the agents operators in the Minecraft world.""" print('Making agents operators...') # Construct path to example tasks relative to project_root example_task_path = os.path.join(project_root, "tasks/example_tasks.json") cmd = f"node {os.path.join(project_root, 'main.js')} --task_path {example_task_path} --task_id debug_{len(agent_names)}_agent_timeout" subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"], cwd=project_root) time.sleep(30) subprocess.run(["tmux", "send-keys", "-t", "server_" + session_name, f"/op @a", "C-m"]) # Check ops file inside the correct tasks/server_data/X directory ops_file_path = os.path.join(tasks_dir, "server_data", session_name, "ops.json") agents_op = check_agent_ops(agent_names, ops_file=ops_file_path) if agents_op: print("Agents are operators! You are good to go :D") else: print("Agents are not operators! Something went wrong :(") make_ops(agent_names, session_name) def check_agent_ops(agent_names, ops_file="ops.json"): """Check if agents are OPs on the server.""" # ops_file path is now provided absolute by caller (make_ops) # if not os.path.isabs(ops_file): # ops_file = os.path.join(project_root, ops_file) # OLD LOGIC if not os.path.exists(ops_file): print(f"Error: ops.json file not found: {ops_file}") return False with open(ops_file, "r") as f: ops_data = json.load(f) ops_names = [op["name"] for op in ops_data] for agent in agent_names: if agent not in ops_names: return False return True def make_script_file_and_run(script_content, file_name, session_name="0", run_in_tmux=True): # Create script inside tasks/tmp/ script_base_dir = os.path.join(tasks_dir, "tmp") os.makedirs(script_base_dir, exist_ok=True) script_abs_path = os.path.join(script_base_dir, file_name) script_dir = os.path.dirname(script_abs_path) # os.makedirs(script_dir, exist_ok=True) # Already handled by script_base_dir creation assert os.path.exists(script_dir), f"Script directory {script_dir} was not created" print(f"Created script directory: {script_dir}") # Call the function before writing the script file with open(script_abs_path, 'w') as f: f.write(script_content) assert os.path.exists(script_abs_path), f"Script file {script_abs_path} was not created" script_file_run = "bash " + script_abs_path # Execute the shell script using subprocess # Run subprocess from project_root so node main.js etc work if run_in_tmux: subprocess.run(["tmux", "send-keys", "-t", session_name, script_file_run, "C-m"], cwd=project_root) else: subprocess.run(script_file_run.split(), cwd=project_root) def make_profiles(agent_names, models, apis, template_profile="profiles/collab_profile.json", url="http://127.0.0.1:8000/v1"): """Generate profile JSON files for each agent.""" # Resolve relative template_profile path relative to project_root if template_profile.startswith("profiles/") and not os.path.isabs(template_profile): template_profile = os.path.join(project_root, template_profile) elif not os.path.isabs(template_profile): # Assume relative to tasks dir if not in profiles/ structure template_profile = os.path.join(tasks_dir, template_profile) with open(template_profile, 'r') as f: content = f.read() profile = json.loads(content) for index in range(len(agent_names)): profile["name"] = agent_names[index] if apis[index] == "vllm": profile["model"] = { "api": "vllm", "model": models[index], "url": url } elif apis[index] == "ollama": profile["model"] = { "api": "ollama", "model": models[index], "embedding": "ollama" } else: profile["model"] = models[index] # Save profiles inside tasks/profiles/ profiles_output_dir = os.path.join(tasks_dir, "profiles") os.makedirs(profiles_output_dir, exist_ok=True) profile_name = f"{agent_names[index]}.json" profile_path = os.path.join(profiles_output_dir, profile_name) with open(profile_path, 'w', encoding='utf-8') as outfile: json.dump(profile, outfile, indent=4) def create_server_files(source_path, num_copies, world_name="Forest"): """Create multiple copies of the server files inside tasks/server_data.""" servers = [] # Define servers list # Ensure source_path is relative to project_root if not absolute if not os.path.isabs(source_path): source_path = os.path.join(project_root, source_path) # Base dir inside tasks/ server_base_dir = os.path.join(tasks_dir, "server_data") os.makedirs(server_base_dir, exist_ok=True) for i in range(num_copies): # Server copies go into tasks/server_data/0/, tasks/server_data/1/, etc. dest_path = os.path.join(server_base_dir, str(i)) copy_server_files(source_path, dest_path) print(dest_path) # Adjust path for edit_file server_prop_path = os.path.join(dest_path, "server.properties") edit_file(server_prop_path, {"server-port": 55916 + i, "level-name": world_name}) # edit_server_properties_file(dest_path, 55916 + i) servers.append((dest_path, 55916 + i)) return servers def edit_file(file, content_dict): try: with open(file, 'r') as f: lines = f.readlines() with open(file, 'w') as f: for line in lines: for key, value in content_dict.items(): if line.startswith(key): f.write(f"{key}={value}\n") else: f.write(line) print(f"{file} updated with {content_dict}") except Exception as e: print(f"Error editing file {file}: {e}") def clean_up_server_files(num_copies): """Delete server files from multiple locations within tasks/server_data.""" server_base_dir = os.path.join(tasks_dir, "server_data") for i in range(num_copies): # Target paths like tasks/server_data/0/ dest_path = os.path.join(server_base_dir, str(i)) delete_server_files(dest_path) def copy_server_files(source_path, dest_path): """Copy server files from source to destination (dest assumed relative to tasks_dir if not absolute).""" # Ensure source_path is relative to project_root if not absolute if not os.path.isabs(source_path): source_path = os.path.join(project_root, source_path) # Destination path is now expected inside tasks/server_data/, handled by caller (create_server_files) # if not os.path.isabs(dest_path): # dest_path = os.path.join(project_root, dest_path) # OLD LOGIC if os.path.exists(dest_path): shutil.rmtree(dest_path) try: shutil.copytree(source_path, dest_path) print(f"Server files copied to {dest_path}") except Exception as e: print(f"Error copying server files: {e}") time.sleep(10) same_files = check_same_files(source_path, dest_path) if not same_files: copy_server_files(source_path, dest_path) print("The destination path does not contain all the same files as the source path.") else: print("The destination path contains all the same files as the source path.") def check_same_files(d1, d2): items1 = set(os.listdir(d1)) items2 = set(os.listdir(d2)) if items1 != items2: return False return True def delete_server_files(dest_path): """Delete server files at the destination path (assumed relative to tasks_dir if not absolute).""" # Path is now expected inside tasks/server_data/, handled by callers # if not os.path.isabs(dest_path): # dest_path = os.path.join(project_root, dest_path) # OLD LOGIC if os.path.exists(dest_path): shutil.rmtree(dest_path) if not os.path.exists(dest_path): print("Server files deleted successfully.") # else: # print("Error deleting server files.") # delete_server_files(dest_path) def launch_world(server_path="./server_data/", agent_names=["andy", "jill"], session_name="server", port=55916): """Launch the Minecraft server world (server assumed inside tasks/server_data).""" # Ensure path is relative to tasks_dir if not absolute (expecting tasks/server_data/X) if not os.path.isabs(server_path): server_path = os.path.join(tasks_dir, server_path) ops_file = os.path.join(server_path, "ops.json") # ops.json inside specific server dir check_agent_ops(agent_names, ops_file=ops_file) # Launch server using tmux (cwd should be the server_path itself) java_cmd = f"java -jar server.jar nogui" # Create tmux session for the server subprocess.run(['tmux', 'new-session', '-d', '-s', session_name], check=True) # Send command to the server session, running from its directory subprocess.run(["tmux", "send-keys", "-t", session_name, java_cmd, "C-m"], cwd=server_path) print(f"Launched Minecraft world in session {session_name} from {server_path} on port {port}...") # Add a delay and check if server started time.sleep(20) # Increased delay if not test_server_running(port): print(f"Warning: Server on port {port} didn't seem to start correctly after launch.") def test_server_running(port=55916): host = 'localhost' with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: try: s.connect((host, port)) print("Server is running on port 55916") return True except ConnectionRefusedError: print("Server is not running on port 55916") return False def kill_world(session_name="server"): """Kill the Minecraft world.""" subprocess.run(["tmux", "send-keys", "-t", session_name, "stop", "C-m"]) time.sleep(5) subprocess.run(["tmux", "kill-session", "-t", session_name]) def detach_process(command): """Detach a process using tmux.""" # Assume commands are run from project root if needed elsewhere process = subprocess.Popen(command, shell=True, preexec_fn=os.setsid) # Example, might need cwd def main(): parser = argparse.ArgumentParser(description="Evaluate MindCraft tasks") parser.add_argument("--task_path", type=str, default="tasks/example_tasks.json", help="Path to the task file or directory (relative to project root)") parser.add_argument("--task_ids", type=str, nargs="+", default=None, help="Specific task IDs to run") parser.add_argument("--num_exp", type=int, default=1, help="Number of experiments per task") parser.add_argument("--num_agents", type=int, default=2, help="Number of agents") parser.add_argument("--model", type=str, default="gpt-4o-mini", help="Model name") parser.add_argument("--api", type=str, default="openai", help="API provider") parser.add_argument("--num_parallel", type=int, default=1, help="Number of parallel experiments") parser.add_argument("--s3", action="store_true", help="Use S3 for storage") parser.add_argument("--bucket_name", type=str, default="mindcraft-experiments", help="S3 bucket name") parser.add_argument("--template_profile", type=str, default="profiles/tasks/collab_profile.json", help="Template profile path") parser.add_argument("--insecure_coding", action="store_true", help="Allow insecure coding practices") parser.add_argument("--url", type=str, default="http://127.0.0.1:8000/v1", help="API URL") parser.add_argument("--check_results", action="store_true", help="Only check results in the specified folder") parser.add_argument("--servers", type=str, nargs="+", default=["local"], help="List of server directories (e.g., 0 1 2 for server_data/0, server_data/1, etc.) or 'local' for parallel local runs") parser.add_argument("--exp_name", type=str, default="exp", help="Experiment name prefix") parser.add_argument("--s3_path", type=str, default="", help="S3 path prefix") parser.add_argument("--max_messages", type=int, default=15, help="Maximum messages per agent") parser.add_argument("--num_examples", type=int, default=2, help="Number of examples for few-shot learning") parser.add_argument("--no_pruning", action="store_true", help="Disable pruning") parser.add_argument("--block_conversation", action="store_true", help="Block agent conversation actions") parser.add_argument("--run_in_tmux", action="store_false", help="Run experiment directly without tmux") # Default is True args = parser.parse_args() # Resolve relative paths provided as arguments or defaults (relative to project root) if not os.path.isabs(args.task_path): args.task_path = os.path.join(project_root, args.task_path) if not os.path.isabs(args.template_profile): # Special handling for default profile path relative to project root if args.template_profile.startswith("profiles/"): args.template_profile = os.path.join(project_root, args.template_profile) else: # Assume relative to tasks dir otherwise args.template_profile = os.path.join(tasks_dir, args.template_profile) if args.check_results: # Hardcode check_folder_results to read from project_root/experiments check_dir = os.path.join(project_root, "experiments") check_folder_results(check_dir) return # Default server source path relative to project_root default_server_source = os.path.join(project_root, "server_data") if not args.run_in_tmux: # Assuming this corresponds to needing server files # Pass default_server_source to create_server_files servers = create_server_files(default_server_source, args.num_parallel, world_name="Forest") # Example world name # The rest of the logic might need adjustment if not using tmux else: # Logic for when run_in_tmux is True (perhaps no server creation needed here?) # Or maybe create_server_files should always run? Adjusting based on original logic # Let's assume server files are always needed for parallel runs servers = create_server_files(default_server_source, args.num_parallel, world_name="Forest") # Example world name # delete all server files (now inside tasks/server_data) # The clean_up_server_files function now uses the correct base path clean_up_server_files(args.num_parallel) if hasattr(args, 'add_keys') and args.add_keys: # Check if arg exists before using update_keys_json() launch_parallel_experiments(args.task_path, num_exp=args.num_exp, exp_name=args.exp_name, num_parallel=args.num_parallel, s3=args.s3, bucket_name=args.bucket_name, template_profile=args.template_profile, model=args.model, api=args.api, insecure_coding=args.insecure_coding, num_agents=args.num_agents, url=args.url, max_messages=args.max_messages, num_examples=args.num_examples, no_pruning=args.no_pruning, block_conversation=args.block_conversation, run_in_tmux=not args.run_in_tmux) if __name__ == "__main__": main()