Merge branch 'main' into constructionTaskRevision

.gitignore

@@ -20,9 +20,9 @@ andy_*.json
 jill_*.json
 src/models/logs/*
 server_data/*
-server_data_*/*
 results/*
 tasks/construction_tasks/test_multiagent_construction_tasks.json
 tasks/construction_tasks/train_multiagent_construction_tasks.json
 tasks/construction_tasks/test/**
-tasks/construction_tasks/train/**
+tasks/construction_tasks/train/**
+server_data*

evaluation_script.py

@@ -411,7 +411,7 @@ def make_ops(agent_names, session_name):
     """Make the agents operators in the Minecraft world."""
     print('Making agents operators...')
 
-    cmd = f"node main.js --task_path example_tasks.json --task_id debug_{len(agent_names)}_agent_timeout"
+    cmd = f"node main.js --task_path tasks/example_tasks.json --task_id debug_{len(agent_names)}_agent_timeout"
 
     subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"])
 

requirements.txt

@@ -0,0 +1,6 @@
+boto3==1.37.11
+botocore==1.37.11
+javascript==1!1.2.2
+numpy==1.22.2
+opencv_python==4.10.0.84
+tqdm==4.62.3

settings.js

@@ -13,7 +13,7 @@ export default
     // the base profile is shared by all bots for default prompts/examples/modes
     "base_profile": "./profiles/defaults/survival.json", // also see creative.json, god_mode.json
     "profiles": ((process.env.PROFILES) && JSON.parse(process.env.PROFILES)) || [
-        "./profiles/deepseek.json",
+        "./andy.json",
         // "./profiles/gpt.json",
         // "./profiles/claude.json",
         // "./profiles/gemini.json",
@@ -37,7 +37,7 @@ export default
 
     "allow_insecure_coding": process.env.INSECURE_CODING || false, // allows newAction command and model can write/run code on your computer. enable at own risk
     "blocked_actions" : process.env.BLOCKED_ACTIONS || [] , // commands to disable and remove from docs. Ex: ["!setMode"]
-
+    "blocked_actions" : process.env.BLOCKED_ACTIONS || ["!checkBlueprint", "!checkBlueprintLevel", "!getBlueprint", "!getBlueprintLevel"] , // commands to disable and remove from docs. Ex: ["!setMode"]
     "code_timeout_mins": -1, // minutes code is allowed to run. -1 for no timeout
     "relevant_docs_count": 5, // Parameter: -1 = all, 0 = no references, 5 = five references. If exceeding the maximum, all reference documents are returned.
 
@@ -47,5 +47,5 @@ export default
     "verbose_commands": true, // show full command syntax
     "narrate_behavior": true, // chat simple automatic actions ('Picking up item!')
     "chat_bot_messages": true, // publicly chat messages to other bots
-    "log_all_prompts": false, // log all prompts to console
+    "log_all_prompts": false, // log all prompts to file
 }

src/agent/task_types/cooking_tasks.js

@@ -256,10 +256,11 @@ export class CookingTaskInitiator {
                 // Entrance features
                 const doorZ = startZ + Math.floor(width / 2);
                 await bot.chat(`/setblock ${startX + depth - 1} ${startY} ${doorZ} stone_brick_stairs[facing=west]`);
-                await bot.chat(`/setblock ${startX + depth - 1} ${startY} ${doorZ - 1} stone_bricks`);
+                await bot.chat(`/setblock ${startX + depth} ${startY} ${doorZ} air`);
-                await bot.chat(`/setblock ${startX + depth - 1} ${startY} ${doorZ + 1} stone_bricks`);
+                // await bot.chat(`/setblock ${startX + depth - 1} ${startY} ${doorZ - 1} stone_bricks`);
-                await bot.chat(`/setblock ${startX + depth} ${startY} ${doorZ} oak_door[half=lower,hinge=left,facing=west,powered=false]`);
+                // await bot.chat(`/setblock ${startX + depth - 1} ${startY} ${doorZ + 1} stone_bricks`);
-                await bot.chat(`/setblock ${startX + depth} ${startY + 1} ${doorZ} oak_door[half=upper,hinge=left,facing=west,powered=false]`);
+                // await bot.chat(`/setblock ${startX + depth} ${startY} ${doorZ} oak_door[half=lower,hinge=left,facing=west,powered=false]`);
+                // await bot.chat(`/setblock ${startX + depth} ${startY + 1} ${doorZ} oak_door[half=upper,hinge=left,facing=west,powered=false]`);
 
                 // Roof construction
                 for (let i = 0; i < 3; i++) {

src/agent/tasks.js

@@ -126,25 +126,16 @@ class CookingCraftingTaskValidator {
         this.data = data;
         this.agent = agent;
     } 
-    validate(has_initiated) {
+    validate() {
-        if (has_initiated) {
+        const result = checkItemPresence(this.data, this.agent);
-
+        let score = 0;
-            const result = checkItemPresence(this.data, this.agent);
+        if (result.success) {
-            let score = 0;
+            score = 1;
-            if (result.success) {
-                score = 1;
-            }
-            return {
-                "valid": result.success, 
-                "score": score,
-            };
-        }
-        else {
-            return {
-                "valid": false,
-                "score": 0
-            };
         }
+        return {
+            "valid": result.success, 
+            "score": score,
+        };
     }
 }
 
@@ -202,7 +193,6 @@ export class Task {
 
         this.name = this.agent.name;
         this.available_agents = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name);
-        this.agent_initialized = false;
     }
 
     getAgentGoal() {
@@ -213,7 +203,7 @@ export class Task {
         let add_string = '';
 
         if (this.task_type === 'cooking') {
-            add_string = '\nIn the end, all the food items should be given to one single player.';
+            add_string = '\nIn the end, all the food items should be given to one single bot.';
         }
 
         // If goal is a string, all agents share the same goal
@@ -254,8 +244,12 @@ export class Task {
     isDone() {
         let res = null;
         if (this.validator)
-            res = this.validator.validate(this.agent_initialized);
+            res = this.validator.validate();
         if (res && res.valid) {
+            // Find all the agents and clear their inventories
+            for (let agent of this.available_agents) {
+                this.agent.bot.chat(`/clear ${agent}`);
+            }
             return {"message": 'Task successful', "score": res.score};
         }
         let other_names = this.available_agents.filter(n => n !== this.name);
@@ -326,8 +320,6 @@ export class Task {
             await new Promise((resolve) => setTimeout(resolve, 500));
         }
 
-        this.agent_initialized = true;
-
         if (this.initiator) {
             await this.initiator.init();
         }
@@ -356,9 +348,9 @@ export class Task {
             await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`);
         }
 
-        const agentGoal = this.getAgentGoal();
+        let agentGoal = this.getAgentGoal();
-        console.log(`Agent goal for agent Id ${this.agent.count_id}: ${agentGoal}`);
         if (agentGoal) {
+            agentGoal += "You have to collaborate with other agents/bots, namely " + this.available_agents.filter(n => n !== this.name).join(', ') + " to complete the task as soon as possible by dividing the work among yourselves.";
             console.log(`Setting goal for agent ${this.agent.count_id}: ${agentGoal}`);
             await executeCommand(this.agent, `!goal("${agentGoal}")`);
         }

src/models/prompter.js

@@ -131,6 +131,8 @@ export class Prompter {
                 profile.api = 'google';
             else if (profile.model.includes('openrouter/'))
                 profile.api = 'openrouter'; // must do before others bc shares model names
+            else if (profile.model.includes('vllm/'))
+                profile.api = 'vllm';
             else if (profile.model.includes('gpt') || profile.model.includes('o1')|| profile.model.includes('o3'))
                 profile.api = 'openai';
             else if (profile.model.includes('claude'))
@@ -304,51 +306,7 @@ export class Prompter {
         this.last_prompt_time = Date.now();
     }
 
-    // async promptConvo(messages) {
-    //     this.most_recent_msg_time = Date.now();
-    //     let current_msg_time = this.most_recent_msg_time;
-    //     for (let i = 0; i < 3; i++) { // try 3 times to avoid hallucinations
-    //         await this.checkCooldown();
-    //         if (current_msg_time !== this.most_recent_msg_time) {
-    //             return '';
-    //         }
-    //         let prompt = this.profile.conversing;
-    //         prompt = await this.replaceStrings(prompt, messages, this.convo_examples);
-    //         let generation = await this.chat_model.sendRequest(messages, prompt);
-    //         // in conversations >2 players LLMs tend to hallucinate and role-play as other bots
-    //         // the FROM OTHER BOT tag should never be generated by the LLM
-    //         if (generation.includes('(FROM OTHER BOT)')) {
-    //             console.warn('LLM hallucinated message as another bot. Trying again...');
-    //             continue;
-    //         }
-    //         if (current_msg_time !== this.most_recent_msg_time) {
-    //             console.warn(this.agent.name + ' received new message while generating, discarding old response.');
-    //             return '';
-    //         }
-    //         return generation;
-    //     }
-    //     return '';
-    // }
-
-    async saveToFile(logFile, logEntry) {
-        let task_id = this.agent.task.task_id;
-        console.log(task_id)
-        let logDir;
-        if (this.task_id === null) {
-            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs`);
-        } else {
-            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs/${task_id}`);
-        }
-
-        await fs.mkdir(logDir, { recursive: true });
-
-        logFile = path.join(logDir, logFile);
-        await fs.appendFile(logFile, String(logEntry), 'utf-8');
-    }
-
     async promptConvo(messages) {
-        // console.log(`[${new Date().toISOString()}] promptConvo called with messages:`, messages);
-
         this.most_recent_msg_time = Date.now();
         let current_msg_time = this.most_recent_msg_time;
 
@@ -371,15 +329,7 @@ export class Prompter {
                     throw new Error('Generated response is not a string');
                 }
                 console.log("Generated response:", generation); 
-                const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+                await this._saveLog(prompt, messages, generation, 'conversation');
-                let logEntry;
-                if (this.task_id === null) {
-                    logEntry = `[${timestamp}] \nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\nResponse:\n${generation}\n\n`;
-                } else {
-                    logEntry = `[${timestamp}] Task ID: ${task_id}\nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\nResponse:\n${generation}\n\n`;
-                }
-                const logFile = `conversation_${timestamp}.txt`;
-                await this.saveToFile(logFile, logEntry);
 
             } catch (error) {
                 console.error('Error during message generation or file writing:', error);
@@ -418,18 +368,9 @@ export class Prompter {
         let prompt = this.profile.coding;
         prompt = await this.replaceStrings(prompt, messages, this.coding_examples);
 
-        let logEntry;
-        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-        if (this.task_id === null) {
-            logEntry = `[${timestamp}] \nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\n`;
-        } else {
-            logEntry = `[${timestamp}] Task ID: ${this.agent.task.task_id}\nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\n`;
-        }
-
-        const logFile = `coding_${timestamp}.txt`;
-        await this.saveToFile(logFile, logEntry);
         let resp = await this.code_model.sendRequest(messages, prompt);
         this.awaiting_coding = false;
+        await this._saveLog(prompt, messages, resp, 'coding');
         return resp;
     }
 
@@ -437,23 +378,13 @@ export class Prompter {
         await this.checkCooldown();
         let prompt = this.profile.saving_memory;
         prompt = await this.replaceStrings(prompt, null, null, to_summarize);
-        let logEntry;
+        let resp = await this.chat_model.sendRequest([], prompt);
-        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        await this._saveLog(prompt, null, resp, 'memSaving');
-        if (this.task_id === null) {
+        if (resp?.includes('</think>')) {
-            logEntry = `[${timestamp}] \nPrompt:\n${prompt}\n\nTo Summarize:\n${JSON.stringify(messages, null, 2)}\n\n`;
+            const [_, afterThink] = resp.split('</think>')
-        } else {
+            resp = afterThink
-            logEntry = `[${timestamp}] Task ID: ${this.agent.task.task_id}\nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(to_summarize, null, 2)}\n\n`;
         }
-        const logFile = `memSaving_${timestamp}.txt`;
+        return resp;
-        await this.saveToFile(logFile, logEntry);
-
-        let generation = await this.chat_model.sendRequest([], prompt);
-        if (generation?.includes('</think>')) {
-            const [_, afterThink] = generation.split('</think>')
-            generation = afterThink
-        }
-        
-        return generation;
     }
 
     async promptShouldRespondToBot(new_message) {
@@ -467,6 +398,7 @@ export class Prompter {
     }
 
     async promptGoalSetting(messages, last_goals) {
+        // deprecated
         let system_message = this.profile.goal_setting;
         system_message = await this.replaceStrings(system_message, messages);
 
@@ -491,4 +423,36 @@ export class Prompter {
         goal.quantity = parseInt(goal.quantity);
         return goal;
     }
+
+    async _saveLog(prompt, messages, generation, tag) {
+        if (!settings.log_all_prompts)
+            return;
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        let logEntry;
+        let task_id = this.agent.task.task_id;
+        if (task_id == null) {
+            logEntry = `[${timestamp}] \nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\nResponse:\n${generation}\n\n`;
+        } else {
+            logEntry = `[${timestamp}] Task ID: ${task_id}\nPrompt:\n${prompt}\n\nConversation:\n${JSON.stringify(messages, null, 2)}\n\nResponse:\n${generation}\n\n`;
+        }
+        const logFile = `${tag}_${timestamp}.txt`;
+        await this._saveToFile(logFile, logEntry);
+    }
+
+    async _saveToFile(logFile, logEntry) {
+        let task_id = this.agent.task.task_id;
+        let logDir;
+        if (task_id == null) {
+            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs`);
+        } else {
+            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs/${task_id}`);
+        }
+
+        await fs.mkdir(logDir, { recursive: true });
+
+        logFile = path.join(logDir, logFile);
+        await fs.appendFile(logFile, String(logEntry), 'utf-8');
+    }
+
+
 }

src/models/vllm.js

@@ -3,6 +3,7 @@
 
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
+import { strictFormat } from '../utils/text.js';
 
 export class VLLM {
     constructor(model_name, url) {
@@ -22,6 +23,11 @@ export class VLLM {
 
     async sendRequest(turns, systemMessage, stop_seq = '***') {
         let messages = [{ 'role': 'system', 'content': systemMessage }].concat(turns);
+        
+        if (this.model_name.includes("deepseek") || this.model_name.inclues("qwen")) {
+            messages = strictFormat(messages);
+        } 
+        
 
         const pack = {
             model: this.model_name || "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
@@ -51,4 +57,20 @@ export class VLLM {
         return res;
     }
 
+    async saveToFile(logFile, logEntry) {
+        let task_id = this.agent.task.task_id;
+        console.log(task_id)
+        let logDir;
+        if (this.task_id === null) {
+            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs`);
+        } else {
+            logDir = path.join(__dirname, `../../bots/${this.agent.name}/logs/${task_id}`);
+        }
+
+        await fs.mkdir(logDir, { recursive: true });
+
+        logFile = path.join(logDir, logFile);
+        await fs.appendFile(logFile, String(logEntry), 'utf-8');
+    }
+
 }

tasks/run_task_file.py

@@ -0,0 +1,58 @@
+# run all tasks in a given file
+
+import os
+import json
+import argparse
+import subprocess
+import time
+
+def run_task(task_path, task_id, profiles=None):
+    """Run a single task using main.js"""
+    # Convert task_path to absolute path if it's relative
+    if not os.path.isabs(task_path):
+        task_path = os.path.abspath(task_path)
+    
+    cmd = ["node", "main.js", "--task_path", task_path, "--task_id", task_id]
+    
+    # Add profiles if provided
+    if profiles:
+        cmd.extend(["--profiles", *profiles])
+    
+    print(f"Running task: {task_id}")
+    
+    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    
+    # Execute the command from the project root directory
+    process = subprocess.run(cmd, check=True, cwd=project_root)
+    
+    return process.returncode == 0
+
+def main():
+    parser = argparse.ArgumentParser(description='Run all tasks in a JSON file sequentially')
+    parser.add_argument('--task_path', required=True, help='Path to the task file')
+    parser.add_argument('--profiles', nargs='+', help='List of agent profile paths')
+    parser.add_argument('--delay', type=int, default=2, help='Delay in seconds between tasks')
+    
+    args = parser.parse_args()
+    
+    # Load the task file
+    with open(args.task_path, 'r') as f:
+        tasks = json.load(f)
+    
+    print(f"Found {len(tasks)} tasks in {args.task_path}")
+    
+    # Run each task sequentially
+    successful_tasks = 0
+    for task_id in tasks:
+        success = run_task(args.task_path, task_id, args.profiles)
+        if success:
+            successful_tasks += 1
+            
+        # Wait between tasks
+        time.sleep(args.delay)
+    
+    print(f"Completed {successful_tasks}/{len(tasks)} tasks successfully")
+
+if __name__ == "__main__":
+    main()
+