mirrors/mindcraft
1
0
Fork 0
mirror of https://github.com/kolbytn/mindcraft.git synced 2025-07-25 17:35:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/main' into stt-new-ollama-model-always-active-vision

Browse source
This commit is contained in:
Sweaterdog 2025-06-18 13:49:30 -07:00
commit 2818330d57
47 changed files with 1247 additions and 1492 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -26,4 +26,5 @@ tasks/construction_tasks/train_multiagent_construction_tasks.json
tasks/construction_tasks/test/**
tasks/construction_tasks/train/**
server_data*
**/.DS_Store
**/.DS_Store
src/mindcraft-py/__pycache__/

8
README.md
View file

@ -10,15 +10,15 @@ Do not connect this bot to public servers with coding enabled. This project allo
## Requirements
- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.20.4)
- [Node.js Installed](https://nodejs.org/) (at least v14)
- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.21.1)
- [Node.js Installed](https://nodejs.org/) (at least v18)
- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) | [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). | [Mistral API Key](https://docs.mistral.ai/getting-started/models/models_overview/) | [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | [Novita AI API Key](https://novita.ai/settings?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link#key-management) |
## Install and Run
1. Make sure you have the requirements above. If you plan to use the STT (Speech-to-Text) feature, also review the "Installation Prerequisites" section regarding `naudiodon`.
2. Clone or download this repository (big green button)
2. Clone or download this repository (big green button) 'git clone https://github.com/kolbytn/mindcraft.git'
3. Rename `keys.example.json` to `keys.json` and fill in your API keys (you only need one). The desired model is set in `andy.json` or other profiles. For other models refer to the table below.
@ -28,7 +28,7 @@ Do not connect this bot to public servers with coding enabled. This project allo
6. Run `node main.js` from the installed directory
If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues.
If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues. To run tasks please refer to [Minecollab Instructions](minecollab.md#installation)
## Tasks

74
main.js
View file

@ -1,9 +1,7 @@
import { AgentProcess } from './src/process/agent_process.js';
import * as Mindcraft from './src/mindcraft/mindcraft.js';
import settings from './settings.js';
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import { createMindServer } from './src/server/mind_server.js';
import { mainProxy } from './src/process/main_proxy.js';
import { readFileSync } from 'fs';
import { initTTS } from './src/process/tts_process.js';
@ -25,36 +23,52 @@ function parseArguments() {
.alias('help', 'h')
.parse();
}
function getProfiles(args) {
return args.profiles || settings.profiles;
const args = parseArguments();
if (args.profiles) {
settings.profiles = args.profiles;
}
async function main() {
if (settings.host_mindserver) {
const mindServer = createMindServer(settings.mindserver_port);
if (args.task_path) {
let tasks = JSON.parse(readFileSync(args.task_path, 'utf8'));
if (args.task_id) {
settings.task = tasks[args.task_id];
settings.task.task_id = args.task_id;
}
mainProxy.connect();
const args = parseArguments();
const profiles = getProfiles(args);
console.log(profiles);
const { load_memory, init_message } = settings;
for (let i=0; i<profiles.length; i++) {
const agent_process = new AgentProcess();
const profile = readFileSync(profiles[i], 'utf8');
const agent_json = JSON.parse(profile);
mainProxy.registerAgent(agent_json.name, agent_process);
agent_process.start(profiles[i], load_memory, init_message, i, args.task_path, args.task_id);
await new Promise(resolve => setTimeout(resolve, 1000));
else {
throw new Error('task_id is required when task_path is provided');
}
initTTS();
}
try {
main();
} catch (error) {
console.error('An error occurred:', error);
process.exit(1);
// these environment variables override certain settings
if (process.env.MINECRAFT_PORT) {
settings.port = process.env.MINECRAFT_PORT;
}
if (process.env.MINDSERVER_PORT) {
settings.mindserver_port = process.env.MINDSERVER_PORT;
}
if (process.env.PROFILES && JSON.parse(process.env.PROFILES).length > 0) {
settings.profiles = JSON.parse(process.env.PROFILES);
}
if (process.env.INSECURE_CODING) {
settings.allow_insecure_coding = true;
}
if (process.env.BLOCKED_ACTIONS) {
settings.blocked_actions = JSON.parse(process.env.BLOCKED_ACTIONS);
}
if (process.env.MAX_MESSAGES) {
settings.max_messages = process.env.MAX_MESSAGES;
}
if (process.env.NUM_EXAMPLES) {
settings.num_examples = process.env.NUM_EXAMPLES;
}
if (process.env.LOG_ALL) {
settings.log_all_prompts = process.env.LOG_ALL;
}
Mindcraft.init(false, settings.mindserver_port);
for (let profile of settings.profiles) {
const profile_json = JSON.parse(readFileSync(profile, 'utf8'));
settings.profile = profile_json;
Mindcraft.createAgent(settings);
}
initTTS();

27
minecollab.md
View file

@ -41,8 +41,16 @@ You can view the crafting task in action [here](https://www.youtube.com/shorts/V
## Installation
You **DO NOT** need Linux to run this, you can run on Windows with the --no-launch-world flag and by installing git bash.
Please follow the installation docs in the README to install mindcraft. You can create a docker image using the Dockerfile.
If you don't own Minecraft you can run a limited version solely for offline games using these instructions:
1. Download the TLauncher https://tlauncher.org/en/
2. Enter a username and select version 1.21.1
3. Click "Multiplayer" and then "Direct Connection"
4. Then enter "localhost:55916" and hit `Join Server`
Download the relevant task files and server data files, you can find the link [here](https://drive.google.com/drive/folders/1XygbitBBTsNO6q_doEiZHmdETpnyRmCS). The tasks files are for specifying the tasks to run and the server data is for allowing the models to launch the task in the correct world automatically. **Unzip the server_data.zip in the base `tasks/` folder**.
Then, set up your conda environment:
@ -55,9 +63,22 @@ pip install -r requirements.txt
Then, you can run the evaluation_script **from the project root** using `python tasks/evaluation_script.py --task_path {your-task-path} --model {model you want to use}`.
### Tmux Installation
**MacOS**:
1. If brew isn't already installed run `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
2. `brew install tmux`
**Linux**: `apt-get -y install tmux`
**Windows**: You can not use tmux on Windows, but you can run tasks with the --no-launch-world flag. Run
```
cd /tasks/server_data/
java -jar server.jar
```
If you want to run with vllm be sure to run with `--api vllm --url {your_url_for_vllm} --model {model_name}`, by default vllm will use http://127.0.0.1:8000/v1 as the url for quering the model!
When running with construction tasks, make sure to set the flag `--insecure_coding` so that the agents can be allowed to write freeform javascript code to complete the tasks. However, when using insecure coding it is highly recommended to use a docker container to avoid damage to your computer.
When running with construction tasks, make sure to set the flag `--insecure_coding` so that the agents can be allowed to write freeform javascript code to complete the tasks. However, when using insecure coding it is **highly recommended** to use a docker container to avoid damage to your computer.
When running an experiment that requires more than 2 agents, use the `--num_agents` flag to match the number of agents in your task file. For example, if you are running a task file with 3 agents, use `--num_agents 3`.
@ -81,7 +102,7 @@ python tasks/evaluation_script.py --task_path {path_to_two_agent_construction_ta
When you launch the evaluation script, you will see the minecraft server being launched. If you want to join this world, you can connect to it on the port localhost:55916 the way you would a standard Minecraft world (go to single player -> direct connection -> type in localhost:55916) It may take a few minutes for everything to be properly loaded - as first the agents need to be added to the world and given the correct permissions to use cheats and add inventory. After about 5 minutes everything should be loaded and working. If you wish to kill the experiment run `tmux kill-server`. Sometimes there will be issues copying the files, if this happens you can run the python file twice.
## Installation (without tmux)
## Windows Installation (without tmux)
If you are on a machine that can't run tmux (like a Windows PC without WSL) or you don't care about doing evaluations only running tasks you can run the following script
@ -99,7 +120,7 @@ As you run, the evalaution script will evaluate the performance so far. It will
### Running multiple worlds in parallel
You can use `--num_parallel` to run multiple Minecraft worlds in parallel. This will launch `n` tmux shells, claled `server_i` and shell `i`, where `i` corresponds to ith parallel world. It will also copy worlds into `server_data_i` as well. On an M3 Mac with 34 GB of RAM, we can normally support up to 4 parallel worlds. When running an open source model, it is more likely you will be constrained by the throughput and size of your GPU RAM. On a cluster of 8 H100s you can expect to run 4 experiments in parallel. However, for best performance it is advisable to only use one parallel world.
You can use `--num_parallel` to run multiple Minecraft worlds in parallel. This will launch `n` tmux shells, called `server_i` and shell `i`, where `i` corresponds to ith parallel world. It will also copy worlds into `server_data_i` as well. On an M3 Mac with 34 GB of RAM, we can normally support up to 4 parallel worlds. When running an open source model, it is more likely you will be constrained by the throughput and size of your GPU RAM. On a cluster of 8 H100s you can expect to run 4 experiments in parallel. However, for best performance it is advisable to only use one parallel world.
### Using an S3 Bucket to store files
To use S3 set the --s3 flag and the --bucket_name to use an s3 bucket to log all the files collected. It will also copy the /bots folder in this case with all of the files in there.

2
package.json
View file

@ -11,7 +11,7 @@
"google-translate-api-x": "^10.7.1",
"groq-sdk": "^0.5.0",
"minecraft-data": "^3.78.0",
"mineflayer": "^4.26.0",
"mineflayer": "^4.29.0",
"mineflayer-armor-manager": "^2.0.1",
"mineflayer-auto-eat": "^3.3.6",
"mineflayer-collectblock": "^1.4.1",

1
requirements.txt
View file

@ -3,3 +3,4 @@ botocore==1.37.11
pandas==2.2.3
prettytable==3.16.0
tqdm==4.62.3
python-socketio[client]

31
settings.js
View file

@ -5,12 +5,9 @@ const settings = {
"auth": "offline", // or "microsoft"
// the mindserver manages all agents and hosts the UI
"host_mindserver": true, // if true, the mindserver will be hosted on this machine. otherwise, specify a public IP address
"mindserver_host": "localhost",
"mindserver_port": 8080,
// the base profile is shared by all bots for default prompts/examples/modes
"base_profile": "./profiles/defaults/survival.json", // also see creative.json, god_mode.json
"base_profile": "survival", // survival, creative, or god_mode
"profiles": [
"./andy.json",
// "./profiles/gpt.json",
@ -26,11 +23,12 @@ const settings = {
// using more than 1 profile requires you to /msg each bot indivually
// individual profiles override values from the base profile
],
"load_memory": false, // load memory from previous session
"init_message": "Respond with hello world and your name", // sends to all on spawn
"only_chat_with": [], // users that the bots listen to and send general messages to. if empty it will chat publicly
"language": "en", // translate to/from this language. Supports these language names: https://cloud.google.com/translate/docs/languages
"show_bot_views": false, // show bot's view in browser at localhost:3000, 3001...
"render_bot_view": false, // show bot's view in browser at localhost:3000, 3001...
"allow_insecure_coding": false, // allows newAction command and model can write/run code on your computer. enable at own risk
"allow_vision": false, // allows vision model to interpret screenshots as inputs
@ -67,27 +65,4 @@ const settings = {
}
// these environment variables override certain settings
if (process.env.MINECRAFT_PORT) {
settings.port = process.env.MINECRAFT_PORT;
}
if (process.env.MINDSERVER_PORT) {
settings.mindserver_port = process.env.MINDSERVER_PORT;
}
if (process.env.PROFILES && JSON.parse(process.env.PROFILES).length > 0) {
settings.profiles = JSON.parse(process.env.PROFILES);
}
if (process.env.INSECURE_CODING) {
settings.allow_insecure_coding = true;
}
if (process.env.BLOCKED_ACTIONS) {
settings.blocked_actions = JSON.parse(process.env.BLOCKED_ACTIONS);
}
if (process.env.MAX_MESSAGES) {
settings.max_messages = process.env.MAX_MESSAGES;
}
if (process.env.NUM_EXAMPLES) {
settings.num_examples = process.env.NUM_EXAMPLES;
}
export default settings;

38
src/agent/agent.js
View file

@ -15,13 +15,13 @@ import { SelfPrompter } from './self_prompter.js';
import convoManager from './conversation.js';
import { handleTranslation, handleEnglishTranslation } from '../utils/translator.js';
import { addBrowserViewer } from './vision/browser_viewer.js';
import settings from '../../settings.js';
import { serverProxy } from './agent_proxy.js';
import { serverProxy } from './mindserver_proxy.js';
import settings from './settings.js';
import { Task } from './tasks/tasks.js';
import { say } from './speak.js';
export class Agent {
async start(profile_fp, load_mem=false, init_message=null, count_id=0, task_path=null, task_id=null) {
async start(load_mem=false, init_message=null, count_id=0) {
this.last_sender = null;
// Safely attach agent instance to a global-like object so STT code can access it.
// This works in Node.js ESM or CommonJS. If "global" doesn't exist, fallback to "globalThis".
@ -41,25 +41,17 @@ export class Agent {
console.log('Starting agent initialization with profile:', profile_fp);
// Initialize components with more detailed error handling
console.log('Initializing action manager...');
this.actions = new ActionManager(this);
console.log('Initializing prompter...');
this.prompter = new Prompter(this, profile_fp);
this.prompter = new Prompter(this, settings.profile);
this.name = this.prompter.getName();
console.log('Initializing history...');
console.log(`Initializing agent ${this.name}...`);
this.history = new History(this);
console.log('Initializing coder...');
this.coder = new Coder(this);
console.log('Initializing npc controller...');
this.npc = new NPCContoller(this);
console.log('Initializing memory bank...');
this.memory_bank = new MemoryBank();
console.log('Initializing self prompter...');
this.self_prompter = new SelfPrompter(this);
convoManager.initAgent(this);
console.log('Initializing examples...');
await this.prompter.initExamples();
console.log('Initializing task...');
// load mem first before doing task
let save_data = null;
@ -72,19 +64,15 @@ export class Agent {
} else {
taskStart = Date.now();
}
this.task = new Task(this, task_path, task_id, taskStart);
this.task = new Task(this, settings.task, taskStart);
this.blocked_actions = settings.blocked_actions.concat(this.task.blocked_actions || []);
blacklistCommands(this.blocked_actions);
serverProxy.connect(this);
console.log(this.name, 'logging into minecraft...');
this.bot = initBot(this.name);
initModes(this);
this.bot.on('login', () => {
console.log(this.name, 'logged in!');
serverProxy.login();
@ -103,6 +91,8 @@ export class Agent {
try {
clearTimeout(spawnTimeout);
addBrowserViewer(this.bot, count_id);
console.log('Initializing vision intepreter...');
this.vision_interpreter = new VisionInterpreter(this, settings.allow_vision);
// wait for a bit so stats are not undefined
await new Promise((resolve) => setTimeout(resolve, 1000));
@ -114,13 +104,13 @@ export class Agent {
this.startEvents();
if (!load_mem) {
if (task_path !== null) {
if (settings.task) {
this.task.initBotTask();
this.task.setAgentGoal();
}
} else {
// set the goal without initializing the rest of the task
if (task_path !== null) {
if (settings.task) {
this.task.setAgentGoal();
}
}
@ -248,8 +238,12 @@ export class Agent {
this.respondFunc = respondFunc;
this.bot.on('whisper', respondFunc);
if (settings.profiles.length === 1)
this.bot.on('chat', respondFunc);
this.bot.on('chat', (username, message) => {
if (serverProxy.getNumOtherAgents() > 0) return;
// only respond to open chat messages when there are no other agents
respondFunc(username, message);
});
// Set up auto-eat
this.bot.autoEat.options = {

73
src/agent/agent_proxy.js
View file

@ -1,73 +0,0 @@
import { io } from 'socket.io-client';
import convoManager from './conversation.js';
import settings from '../../settings.js';
class AgentServerProxy {
constructor() {
if (AgentServerProxy.instance) {
return AgentServerProxy.instance;
}
this.socket = null;
this.connected = false;
AgentServerProxy.instance = this;
}
connect(agent) {
if (this.connected) return;
this.agent = agent;
this.socket = io(`http://${settings.mindserver_host}:${settings.mindserver_port}`);
this.connected = true;
this.socket.on('connect', () => {
console.log('Connected to MindServer');
});
this.socket.on('disconnect', () => {
console.log('Disconnected from MindServer');
this.connected = false;
});
this.socket.on('chat-message', (agentName, json) => {
convoManager.receiveFromBot(agentName, json);
});
this.socket.on('agents-update', (agents) => {
convoManager.updateAgents(agents);
});
this.socket.on('restart-agent', (agentName) => {
console.log(`Restarting agent: ${agentName}`);
this.agent.cleanKill();
});
this.socket.on('send-message', (agentName, message) => {
try {
this.agent.respondFunc("NO USERNAME", message);
} catch (error) {
console.error('Error: ', JSON.stringify(error, Object.getOwnPropertyNames(error)));
}
});
}
login() {
this.socket.emit('login-agent', this.agent.name);
}
shutdown() {
this.socket.emit('shutdown');
}
getSocket() {
return this.socket;
}
}
// Create and export a singleton instance
export const serverProxy = new AgentServerProxy();
export function sendBotChatToServer(agentName, json) {
serverProxy.getSocket().emit('chat-message', agentName, json);
}

4
src/agent/coder.js
View file

@ -1,6 +1,5 @@
import { writeFile, readFile, mkdirSync } from 'fs';
import settings from '../../settings.js';
import { makeCompartment } from './library/lockdown.js';
import { makeCompartment, lockdown } from './library/lockdown.js';
import * as skills from './library/skills.js';
import * as world from './library/world.js';
import { Vec3 } from 'vec3';
@ -27,6 +26,7 @@ export class Coder {
async generateCode(agent_history) {
this.agent.bot.modes.pause('unstuck');
lockdown();
// this message history is transient and only maintained in this function
let messages = agent_history.getHistory();
messages.push({role: 'system', content: 'Code generation started. Write code in codeblock in your response:'});

4
src/agent/commands/actions.js
View file

@ -1,5 +1,5 @@
import * as skills from '../library/skills.js';
import settings from '../../../settings.js';
import settings from '../settings.js';
import convoManager from '../conversation.js';
@ -46,7 +46,7 @@ export const actionsList = [
result = 'Error generating code: ' + e.toString();
}
};
await agent.actions.runAction('action:newAction', actionFn);
await agent.actions.runAction('action:newAction', actionFn, {timeout: settings.code_timeout_mins});
return result;
}
},

7
src/agent/conversation.js
View file

@ -1,10 +1,9 @@
import settings from '../../settings.js';
import { readFileSync } from 'fs';
import settings from './settings.js';
import { containsCommand } from './commands/index.js';
import { sendBotChatToServer } from './agent_proxy.js';
import { sendBotChatToServer } from './mindserver_proxy.js';
let agent;
let agent_names = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name);
let agent_names = [];
let agents_in_game = [];
class Conversation {

2
src/agent/history.js
View file

@ -1,6 +1,6 @@
import { writeFileSync, readFileSync, mkdirSync, existsSync } from 'fs';
import { NPCData } from './npc/data.js';
import settings from '../../settings.js';
import settings from './settings.js';
export class History {

26
src/agent/library/lockdown.js
View file

@ -4,16 +4,22 @@ import 'ses';
// We disable some of the taming to allow for more flexibility
// For configuration, see https://github.com/endojs/endo/blob/master/packages/ses/docs/lockdown.md
lockdown({
// basic devex and quality of life improvements
localeTaming: 'unsafe',
consoleTaming: 'unsafe',
errorTaming: 'unsafe',
stackFiltering: 'verbose',
// allow eval outside of created compartments
// (mineflayer dep "protodef" uses eval)
evalTaming: 'unsafeEval',
});
let lockeddown = false;
export function lockdown() {
if (lockeddown) return;
lockeddown = true;
lockdown({
// basic devex and quality of life improvements
localeTaming: 'unsafe',
consoleTaming: 'unsafe',
errorTaming: 'unsafe',
stackFiltering: 'verbose',
// allow eval outside of created compartments
// (mineflayer dep "protodef" uses eval)
evalTaming: 'unsafeEval',
});
}
export const makeCompartment = (endowments = {}) => {
return new Compartment({

115
src/agent/mindserver_proxy.js Normal file
View file

@ -0,0 +1,115 @@
import { io } from 'socket.io-client';
import convoManager from './conversation.js';
import { setSettings } from './settings.js';
// agents connection to mindserver
// always connect to localhost
class MindServerProxy {
constructor() {
if (MindServerProxy.instance) {
return MindServerProxy.instance;
}
this.socket = null;
this.connected = false;
this.agents = [];
MindServerProxy.instance = this;
}
async connect(name, port) {
if (this.connected) return;
this.name = name;
this.socket = io(`http://localhost:${port}`);
await new Promise((resolve, reject) => {
this.socket.on('connect', resolve);
this.socket.on('connect_error', (err) => {
console.error('Connection failed:', err);
reject(err);
});
});
this.connected = true;
console.log(name, 'connected to MindServer');
this.socket.on('disconnect', () => {
console.log('Disconnected from MindServer');
this.connected = false;
});
this.socket.on('chat-message', (agentName, json) => {
convoManager.receiveFromBot(agentName, json);
});
this.socket.on('agents-update', (agents) => {
this.agents = agents;
convoManager.updateAgents(agents);
if (this.agent?.task) {
console.log(this.agent.name, 'updating available agents');
this.agent.task.updateAvailableAgents(agents);
}
});
this.socket.on('restart-agent', (agentName) => {
console.log(`Restarting agent: ${agentName}`);
this.agent.cleanKill();
});
this.socket.on('send-message', (agentName, message) => {
try {
this.agent.respondFunc("NO USERNAME", message);
} catch (error) {
console.error('Error: ', JSON.stringify(error, Object.getOwnPropertyNames(error)));
}
});
// Request settings and wait for response
await new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Settings request timed out after 5 seconds'));
}, 5000);
this.socket.emit('get-settings', name, (response) => {
clearTimeout(timeout);
if (response.error) {
return reject(new Error(response.error));
}
setSettings(response.settings);
resolve();
});
});
}
setAgent(agent) {
this.agent = agent;
}
getAgents() {
return this.agents;
}
getNumOtherAgents() {
return this.agents.length - 1;
}
login() {
this.socket.emit('login-agent', this.agent.name);
}
shutdown() {
this.socket.emit('shutdown');
}
getSocket() {
return this.socket;
}
}
// Create and export a singleton instance
export const serverProxy = new MindServerProxy();
export function sendBotChatToServer(agentName, json) {
serverProxy.getSocket().emit('chat-message', agentName, json);
}

2
src/agent/modes.js
View file

@ -1,7 +1,7 @@
import * as skills from './library/skills.js';
import * as world from './library/world.js';
import * as mc from '../utils/mcdata.js';
import settings from '../../settings.js'
import settings from './settings.js'
import convoManager from './conversation.js';
async function say(agent, message) {

7
src/agent/settings.js Normal file
View file

@ -0,0 +1,7 @@
// extremely lightweight obj that can be imported/modified by any file
let settings = {};
export default settings;
export function setSettings(new_settings) {
Object.keys(settings).forEach(key => delete settings[key]);
Object.assign(settings, new_settings);
}

52
src/agent/tasks/tasks.js
View file

@ -1,7 +1,6 @@
import { readFileSync , writeFileSync, existsSync} from 'fs';
import { executeCommand } from '../commands/index.js';
import { getPosition } from '../library/world.js';
import settings from '../../../settings.js';
import { ConstructionTaskValidator, Blueprint } from './construction_tasks.js';
import { CookingTaskInitiator } from './cooking_tasks.js';
@ -233,27 +232,26 @@ class CookingCraftingTaskValidator {
}
export class Task {
constructor(agent, task_path, task_id, taskStartTime = null) {
constructor(agent, task_data, taskStartTime = null) {
this.agent = agent;
this.data = null;
if (taskStartTime !== null)
this.taskStartTime = taskStartTime;
else
this.taskStartTime = Date.now();
console.log("Task start time set to", this.taskStartTime);
this.validator = null;
this.reset_function = null;
this.blocked_actions = [];
this.task_id = task_id;
if (task_path && task_id) {
console.log('Starting task', task_id);
if (task_id.endsWith('hells_kitchen')) {
this.task_data = task_data;
if (task_data) {
console.log('Starting task', task_data.task_id);
console.log("Task start time set to", this.taskStartTime);
if (task_data.task_id.endsWith('hells_kitchen')) {
// Reset hells_kitchen progress when a new task starts
hellsKitchenProgressManager.resetTask(task_id);
hellsKitchenProgressManager.resetTask(task_data.task_id);
console.log('Reset Hells Kitchen progress for new task');
}
this.data = this.loadTask(task_path, task_id);
this.data = task_data;
this.task_type = this.data.type;
if (this.task_type === 'construction' && this.data.blueprint) {
this.blueprint = new Blueprint(this.data.blueprint);
@ -300,7 +298,11 @@ export class Task {
}
this.name = this.agent.name;
this.available_agents = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name);
this.available_agents = []
}
updateAvailableAgents(agents) {
this.available_agents = agents
}
// Add this method if you want to manually reset the hells_kitchen progress
@ -360,28 +362,6 @@ export class Task {
return null;
}
loadTask(task_path, task_id) {
try {
const tasksFile = readFileSync(task_path, 'utf8');
const tasks = JSON.parse(tasksFile);
let task = tasks[task_id];
task['task_id'] = task_id;
console.log(task);
console.log(this.agent.count_id);
if (!task) {
throw new Error(`Task ${task_id} not found`);
}
// if ((!task.agent_count || task.agent_count <= 1) && this.agent.count_id > 0) {
// task = null;
// }
return task;
} catch (error) {
console.error('Error loading task:', error);
process.exit(1);
}
}
isDone() {
let res = null;
if (this.validator)
@ -426,7 +406,6 @@ export class Task {
}
async initBotTask() {
await this.setAgentGoal();
await this.agent.bot.chat(`/clear ${this.name}`);
console.log(`Cleared ${this.name}'s inventory.`);
@ -511,7 +490,7 @@ export class Task {
this.agent.killAll();
}
}
await new Promise((resolve) => setTimeout(resolve, 500));
if (this.data.conversation && this.agent.count_id === 0) {
let other_name = this.available_agents.filter(n => n !== this.name)[0];
let waitCount = 0;
@ -526,8 +505,7 @@ export class Task {
}
await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`);
}
await this.setAgentGoal();
}
async teleportBots() {

4
src/agent/vision/browser_viewer.js
View file

@ -1,8 +1,8 @@
import settings from '../../../settings.js';
import settings from '../settings.js';
import prismarineViewer from 'prismarine-viewer';
const mineflayerViewer = prismarineViewer.mineflayer;
export function addBrowserViewer(bot, count_id) {
if (settings.show_bot_views)
if (settings.render_bot_view)
mineflayerViewer(bot, { port: 3000+count_id, firstPerson: true, });
}

27
src/mindcraft-py/example.py Normal file
View file

@ -0,0 +1,27 @@
import mindcraft
import json
import os
# Initialize Mindcraft, starting the Node.js server
# This will also connect to the MindServer via websockets
mindcraft.init()
# Get the directory of the current script
script_dir = os.path.dirname(os.path.abspath(__file__))
profile_path = os.path.abspath(os.path.join(script_dir, '..', '..', 'andy.json'))
# Load agent settings from a JSON file
try:
with open(profile_path, 'r') as f:
profile_data = json.load(f)
settings = {"profile": profile_data}
mindcraft.create_agent(settings)
settings_copy = settings.copy()
settings_copy['profile']['name'] = 'andy2'
mindcraft.create_agent(settings_copy)
except FileNotFoundError:
print(f"Error: Could not find andy.json at {profile_path}")
mindcraft.wait()

24
src/mindcraft-py/init-mindcraft.js Normal file
View file

@ -0,0 +1,24 @@
import * as Mindcraft from '../mindcraft/mindcraft.js';
import settings from '../../settings.js';
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
function parseArguments() {
return yargs(hideBin(process.argv))
.option('mindserver_port', {
type: 'number',
describe: 'Mindserver port',
default: settings.mindserver_port
})
.help()
.alias('help', 'h')
.parse();
}
const args = parseArguments();
settings.mindserver_port = args.mindserver_port;
Mindcraft.init(settings.mindserver_port);
console.log(`Mindcraft initialized with MindServer at localhost:${settings.mindserver_port}`);

99
src/mindcraft-py/mindcraft.py Normal file
View file

@ -0,0 +1,99 @@
import subprocess
import socketio
import time
import json
import os
import atexit
import threading
import sys
import signal
class Mindcraft:
def __init__(self):
self.sio = socketio.Client()
self.process = None
self.connected = False
self.log_thread = None
def _log_reader(self):
for line in iter(self.process.stdout.readline, ''):
sys.stdout.write(f'[Node.js] {line}')
sys.stdout.flush()
def init(self, port=8080):
if self.process:
return
self.port = port
node_script_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'init-mindcraft.js'))
self.process = subprocess.Popen([
'node',
node_script_path,
'--mindserver_port', str(self.port)
], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
self.log_thread = threading.Thread(target=self._log_reader)
self.log_thread.daemon = True
self.log_thread.start()
atexit.register(self.shutdown)
time.sleep(2) # Give server time to start before connecting
try:
self.sio.connect(f'http://localhost:{self.port}')
self.connected = True
print("Connected to MindServer. Mindcraft is initialized.")
except socketio.exceptions.ConnectionError as e:
print(f"Failed to connect to MindServer: {e}")
self.shutdown()
raise
def create_agent(self, settings_json):
if not self.connected:
raise Exception("Not connected to MindServer. Call init() first.")
profile_data = settings_json.get('profile', {})
def callback(response):
if response.get('success'):
print(f"Agent '{profile_data.get('name')}' created successfully")
else:
print(f"Error creating agent: {response.get('error', 'Unknown error')}")
self.sio.emit('create-agent', settings_json, callback=callback)
def shutdown(self):
if self.sio.connected:
self.sio.disconnect()
self.connected = False
if self.process:
self.process.terminate()
self.process.wait()
self.process = None
print("Mindcraft shut down.")
def wait(self):
"""Block the main thread until Ctrl+C is pressed so the server stays up,"""
print("Server is running. Press Ctrl+C to exit.")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("\nCtrl+C detected. Exiting...")
self.shutdown()
mindcraft_instance = Mindcraft()
def init(port=8080):
mindcraft_instance.init(port)
def create_agent(settings_json):
mindcraft_instance.create_agent(settings_json)
def shutdown():
mindcraft_instance.shutdown()
def wait():
mindcraft_instance.wait()

64
src/mindcraft/mindcraft.js Normal file
View file

@ -0,0 +1,64 @@
import { createMindServer, registerAgent } from './mindserver.js';
import { AgentProcess } from '../process/agent_process.js';
let mindserver;
let connected = false;
let agent_processes = {};
let agent_count = 0;
let host = 'localhost';
let port = 8080;
export async function init(host_public=false, port=8080) {
if (connected) {
console.error('Already initiliazed!');
return;
}
mindserver = createMindServer(host_public, port);
port = port;
connected = true;
}
export async function createAgent(settings) {
if (!settings.profile.name) {
console.error('Agent name is required in profile');
return;
}
settings = JSON.parse(JSON.stringify(settings));
let agent_name = settings.profile.name;
registerAgent(settings);
let load_memory = settings.load_memory || false;
let init_message = settings.init_message || null;
const agentProcess = new AgentProcess(agent_name, port);
agentProcess.start(load_memory, init_message, agent_count);
agent_count++;
agent_processes[settings.profile.name] = agentProcess;
}
export function getAgentProcess(agentName) {
return agent_processes[agentName];
}
export function startAgent(agentName) {
if (agent_processes[agentName]) {
agent_processes[agentName].continue();
}
else {
console.error(`Cannot start agent ${agentName}; not found`);
}
}
export function stopAgent(agentName) {
if (agent_processes[agentName]) {
agent_processes[agentName].stop();
}
}
export function shutdown() {
console.log('Shutting down');
for (let agentName in agent_processes) {
agent_processes[agentName].stop();
}
setTimeout(() => {
process.exit(0);
}, 2000);
}

196
src/mindcraft/mindserver.js Normal file
View file

@ -0,0 +1,196 @@
import { Server } from 'socket.io';
import express from 'express';
import http from 'http';
import path from 'path';
import { fileURLToPath } from 'url';
import * as mindcraft from './mindcraft.js';
import { readFileSync } from 'fs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Mindserver is:
// - central hub for communication between all agent processes
// - api to control from other languages and remote users
// - host for webapp
let io;
let server;
const agent_connections = {};
const settings_spec = JSON.parse(readFileSync(path.join(__dirname, 'public/settings_spec.json'), 'utf8'));
class AgentConnection {
constructor(settings) {
this.socket = null;
this.settings = settings;
this.in_game = false;
}
}
export function registerAgent(settings) {
let agentConnection = new AgentConnection(settings);
agent_connections[settings.profile.name] = agentConnection;
}
export function logoutAgent(agentName) {
if (agent_connections[agentName]) {
agent_connections[agentName].in_game = false;
agentsUpdate();
}
}
// Initialize the server
export function createMindServer(host_public = false, port = 8080) {
const app = express();
server = http.createServer(app);
io = new Server(server);
// Serve static files
const __dirname = path.dirname(fileURLToPath(import.meta.url));
app.use(express.static(path.join(__dirname, 'public')));
// Socket.io connection handling
io.on('connection', (socket) => {
let curAgentName = null;
console.log('Client connected');
agentsUpdate(socket);
socket.on('create-agent', (settings, callback) => {
console.log('API create agent...');
for (let key in settings_spec) {
if (!(key in settings)) {
if (settings_spec[key].required) {
callback({ success: false, error: `Setting ${key} is required` });
return;
}
else {
settings[key] = settings_spec[key].default;
}
}
}
for (let key in settings) {
if (!(key in settings_spec)) {
delete settings[key];
}
}
if (settings.profile?.name) {
if (settings.profile.name in agent_connections) {
callback({ success: false, error: 'Agent already exists' });
return;
}
mindcraft.createAgent(settings);
callback({ success: true });
}
else {
console.error('Agent name is required in profile');
callback({ success: false, error: 'Agent name is required in profile' });
}
});
socket.on('get-settings', (agentName, callback) => {
if (agent_connections[agentName]) {
callback({ settings: agent_connections[agentName].settings });
} else {
callback({ error: `Agent '${agentName}' not found.` });
}
});
socket.on('login-agent', (agentName) => {
if (agent_connections[agentName]) {
agent_connections[agentName].socket = socket;
agent_connections[agentName].in_game = true;
curAgentName = agentName;
agentsUpdate();
}
else {
console.warn(`Unregistered agent ${agentName} tried to login`);
}
});
socket.on('disconnect', () => {
if (agent_connections[curAgentName]) {
console.log(`Agent ${curAgentName} disconnected`);
agent_connections[curAgentName].in_game = false;
agentsUpdate();
}
});
socket.on('chat-message', (agentName, json) => {
if (!agent_connections[agentName]) {
console.warn(`Agent ${agentName} tried to send a message but is not logged in`);
return;
}
console.log(`${curAgentName} sending message to ${agentName}: ${json.message}`);
agent_connections[agentName].socket.emit('chat-message', curAgentName, json);
});
socket.on('restart-agent', (agentName) => {
console.log(`Restarting agent: ${agentName}`);
agent_connections[agentName].socket.emit('restart-agent');
});
socket.on('stop-agent', (agentName) => {
mindcraft.stopAgent(agentName);
});
socket.on('start-agent', (agentName) => {
mindcraft.startAgent(agentName);
});
socket.on('stop-all-agents', () => {
console.log('Killing all agents');
for (let agentName in agent_connections) {
mindcraft.stopAgent(agentName);
}
});
socket.on('shutdown', () => {
console.log('Shutting down');
for (let agentName in agent_connections) {
mindcraft.stopAgent(agentName);
}
// wait 2 seconds
setTimeout(() => {
console.log('Exiting MindServer');
process.exit(0);
}, 2000);
});
socket.on('send-message', (agentName, message) => {
if (!agent_connections[agentName]) {
console.warn(`Agent ${agentName} not in game, cannot send message via MindServer.`);
return
}
try {
console.log(`Sending message to agent ${agentName}: ${message}`);
agent_connections[agentName].socket.emit('send-message', agentName, message)
} catch (error) {
console.error('Error: ', error);
}
});
});
let host = host_public ? '0.0.0.0' : 'localhost';
server.listen(port, host, () => {
console.log(`MindServer running on port ${port}`);
});
return server;
}
function agentsUpdate(socket) {
if (!socket) {
socket = io;
}
let agents = [];
for (let agentName in agent_connections) {
agents.push({name: agentName, in_game: agent_connections[agentName].in_game});
};
socket.emit('agents-update', agents);
}
// Optional: export these if you need access to them from other files
export const getIO = () => io;
export const getServer = () => server;

286
src/mindcraft/public/index.html Normal file
View file

@ -0,0 +1,286 @@
<!DOCTYPE html>
<html>
<head>
<title>Mindcraft</title>
<script src="/socket.io/socket.io.js"></script>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
background: #1a1a1a;
color: #e0e0e0;
}
#agents {
background: #2d2d2d;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.2);
}
h1 {
color: #ffffff;
}
.agent {
margin: 10px 0;
padding: 10px;
background: #363636;
border-radius: 4px;
display: flex;
justify-content: space-between;
align-items: center;
}
.restart-btn, .start-btn, .stop-btn {
color: white;
border: none;
padding: 5px 10px;
border-radius: 4px;
cursor: pointer;
margin-left: 5px;
}
.restart-btn {
background: #4CAF50;
}
.start-btn {
background: #2196F3;
}
.stop-btn {
background: #f44336;
}
.restart-btn:hover { background: #45a049; }
.start-btn:hover { background: #1976D2; }
.stop-btn:hover { background: #d32f2f; }
.status-icon {
font-size: 12px;
margin-right: 8px;
}
.status-icon.online {
color: #4CAF50;
}
.status-icon.offline {
color: #f44336;
}
#settingsForm {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
gap: 8px;
margin-top: 10px;
}
.setting-wrapper {
display: flex;
align-items: center;
gap: 6px;
background: #3a3a3a;
padding: 6px 8px;
border-radius: 4px;
width: 100%;
box-sizing: border-box;
min-width: 0;
}
.setting-wrapper label {
flex: 0 0 50%;
font-size: 0.9em;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.setting-wrapper input[type="text"],
.setting-wrapper input[type="number"] {
flex: 1 1 0;
background: #262626;
border: 1px solid #555;
color: #e0e0e0;
border-radius: 4px;
padding: 4px 6px;
max-width: 100%;
min-width: 0;
}
.setting-wrapper input[type="checkbox"] {
transform: scale(1.2);
}
.agent-viewer {
width: 200px;
height: 150px;
border: none;
margin-left: 10px;
}
.start-btn:disabled {
opacity: 0.4;
cursor: not-allowed;
}
.agent-view-container {
margin-top: 6px;
display: flex;
justify-content: flex-start;
}
</style>
</head>
<body>
<h1>Mindcraft</h1>
<div id="agents"></div>
<div id="createAgentSection" style="margin-top:20px;background:#2d2d2d;padding:20px;border-radius:8px;">
<h2>Create Agent</h2>
<div id="settingsForm"></div>
<div id="profileStatus" style="margin-top:6px;font-style:italic;color:#cccccc;">Profile: Not uploaded</div>
<div style="margin-top:10px;">
<button id="uploadProfileBtn" class="start-btn">Upload Profile</button>
<input type="file" id="profileFileInput" accept=".json,application/json" style="display:none">
<button id="submitCreateAgentBtn" class="start-btn" disabled>Create Agent</button>
</div>
<div id="createError" style="color:#f44336;margin-top:10px;"></div>
</div>
<script>
const socket = io();
const agentsDiv = document.getElementById('agents');
let settingsSpec = {};
let profileData = null;
const agentSettings = {};
fetch('/settings_spec.json')
.then(r => r.json())
.then(spec => {
settingsSpec = spec;
const form = document.getElementById('settingsForm');
Object.keys(spec).forEach(key => {
if (key === 'profile') return; // profile handled via upload
const cfg = spec[key];
const wrapper = document.createElement('div');
wrapper.className = 'setting-wrapper';
const label = document.createElement('label');
label.textContent = key;
label.title = cfg.description || '';
let input;
switch (cfg.type) {
case 'boolean':
input = document.createElement('input');
input.type = 'checkbox';
input.checked = cfg.default === true;
break;
case 'number':
input = document.createElement('input');
input.type = 'number';
input.value = cfg.default;
break;
default:
input = document.createElement('input');
input.type = 'text';
input.value = typeof cfg.default === 'object' ? JSON.stringify(cfg.default) : cfg.default;
}
input.title = cfg.description || '';
input.id = `setting-${key}`;
wrapper.appendChild(label);
wrapper.appendChild(input);
form.appendChild(wrapper);
});
});
document.getElementById('uploadProfileBtn').addEventListener('click', () => {
document.getElementById('profileFileInput').click();
});
document.getElementById('profileFileInput').addEventListener('change', e => {
const file = e.target.files[0];
if (!file) return;
const reader = new FileReader();
reader.onload = ev => {
try {
profileData = JSON.parse(ev.target.result);
document.getElementById('submitCreateAgentBtn').disabled = false;
document.getElementById('profileStatus').textContent = `Profile: ${profileData.name || 'Uploaded'}`;
document.getElementById('createError').textContent = '';
} catch (err) {
document.getElementById('createError').textContent = 'Invalid profile JSON: ' + err.message;
profileData = null;
document.getElementById('submitCreateAgentBtn').disabled = true;
document.getElementById('profileStatus').textContent = 'Profile: Not uploaded';
}
};
reader.readAsText(file);
e.target.value = '';
});
document.getElementById('submitCreateAgentBtn').addEventListener('click', () => {
if (!profileData) return;
const settings = { profile: profileData };
Object.keys(settingsSpec).forEach(key => {
if (key === 'profile') return;
const input = document.getElementById(`setting-${key}`);
if (!input) return;
const type = settingsSpec[key].type;
let val;
if (type === 'boolean') val = input.checked;
else if (type === 'number') val = Number(input.value);
else if (type === 'array' || type === 'object') {
try { val = JSON.parse(input.value); }
catch { val = input.value; }
} else val = input.value;
settings[key] = val;
});
socket.emit('create-agent', settings, res => {
if (!res.success) {
document.getElementById('createError').textContent = res.error || 'Unknown error';
} else {
// reset on success
profileData = null;
document.getElementById('submitCreateAgentBtn').disabled = true;
document.getElementById('profileStatus').textContent = 'Profile: Not uploaded';
document.getElementById('createError').textContent = '';
}
});
});
function fetchAgentSettings(name) {
return new Promise((resolve) => {
if (agentSettings[name]) { resolve(agentSettings[name]); return; }
socket.emit('get-settings', name, res => {
if (res.settings) {
agentSettings[name] = res.settings;
resolve(res.settings);
} else resolve(null);
});
});
}
async function renderAgents(agents) {
// fetch settings for any new agents
await Promise.all(agents.map(a => fetchAgentSettings(a.name)));
agentsDiv.innerHTML = agents.length ?
agents.map((agent, idx) => {
const cfg = agentSettings[agent.name] || {};
const showViewer = cfg.render_bot_view === true;
const viewerHTML = showViewer ? `<div class="agent-view-container"><iframe class="agent-viewer" src="http://localhost:${3000 + idx}"></iframe></div>` : '';
return `
<div class="agent">
<div style="display:flex;justify-content:space-between;align-items:center;">
<span><span class="status-icon ${agent.in_game ? 'online' : 'offline'}">●</span>${agent.name}</span>
<div style="display:flex;align-items:center;">
${agent.in_game ? `
<button class="stop-btn" onclick="stopAgent('${agent.name}')">Stop</button>
<button class="restart-btn" onclick="restartAgent('${agent.name}')">Restart</button>
<input type="text" id="messageInput-${agent.name}" placeholder="Enter message..." style="margin-left:4px;">
<button class="start-btn" onclick="sendMessage('${agent.name}', document.getElementById('messageInput-${agent.name}').value)">Send</button>
` : `
<button class="start-btn" onclick="startAgent('${agent.name}')">Start</button>
`}
</div>
</div>
${viewerHTML}
</div>`;
}).join('') +
`<button class="stop-btn" onclick="killAllAgents()">Stop All</button>
<button class="stop-btn" onclick="shutdown()">Shutdown</button>` :
'<div class="agent">No agents connected</div>';
}
socket.on('agents-update', agents => { renderAgents(agents); });
function restartAgent(n) { socket.emit('restart-agent', n); }
function startAgent(n) { socket.emit('start-agent', n); }
function stopAgent(n) { socket.emit('stop-agent', n); }
function killAllAgents() { socket.emit('stop-all-agents'); }
function shutdown() { socket.emit('shutdown'); }
function sendMessage(n, m) { socket.emit('send-message', n, m); }
</script>
</body>
</html>

127
src/mindcraft/public/settings_spec.json Normal file
View file

@ -0,0 +1,127 @@
{
"profile": {
"type": "object",
"required": true,
"description": "The profile object to use, including name, prompts, and examples"
},
"minecraft_version": {
"type": "string",
"description": "The version of Minecraft to use",
"default": "1.21.1"
},
"host": {
"type": "string",
"description": "The minecraft server host address to connect to",
"default": "127.0.0.1"
},
"port": {
"type": "number",
"description": "The minecraft server port to connect to",
"default": 55916
},
"auth": {
"type": "string",
"description": "The authentication method to use",
"default": "offline"
},
"base_profile": {
"type": "string",
"description": "Allowed values: survival, creative, god_mode. Each has fine tuned settings for different game modes.",
"default": "survival"
},
"load_memory": {
"type": "boolean",
"description": "Whether to load bot's previous memory",
"default": false
},
"init_message": {
"type": "string",
"description": "The initial message to send to the bot",
"default": "Respond with hello world and your name"
},
"only_chat_with": {
"type": "array",
"description": "List of agents to only chat with. If empty, the bot will chat publicly",
"default": []
},
"speak": {
"type": "boolean",
"description": "Whether to enable text-to-speech reading on the host machine",
"default": false
},
"language": {
"type": "string",
"description": "The language to automatically translate to and from using google translate",
"default": "en"
},
"allow_vision": {
"type": "boolean",
"description": "Whether to allow vision capabilities",
"default": false
},
"blocked_actions": {
"type": "array",
"description": "List of actions that are blocked",
"default": ["!checkBlueprint", "!checkBlueprintLevel", "!getBlueprint", "!getBlueprintLevel"]
},
"relevant_docs_count": {
"type": "number",
"description": "Number of relevant function documents to include in the prompt for LLM code writing",
"default": 5
},
"max_messages": {
"type": "number",
"description": "Maximum number of recent messages to keep in context for LLM",
"default": 15
},
"num_examples": {
"type": "number",
"description": "Number of examples to select to help prompt better LLM responses",
"default": 2
},
"max_commands": {
"type": "number",
"description": "Maximum number of commands allowed in consecutive responses. -1 for no limit",
"default": -1
},
"narrate_behavior": {
"type": "boolean",
"description": "Whether to openly chat automatic behavior like 'Picking up item!'",
"default": true
},
"log_all_prompts": {
"type": "boolean",
"description": "Whether to log all prompts to file. Can be very verbose.",
"default": false
},
"verbose_commands": {
"type": "boolean",
"description": "Whether to show full command syntax in bot responses. If false will use a shortened syntax.",
"default": true
},
"chat_bot_messages": {
"type": "boolean",
"description": "Whether to publicly chat messages to and from other bots",
"default": true
},
"render_bot_view": {
"type": "boolean",
"description": "Whether to render bot view for user observation. Does not give bot vision.",
"default": false
},
"allow_insecure_coding": {
"type": "boolean",
"description": "Whether to allow newAction command that let's LLM write/run code on host computer. Despite sandboxxing, it is potentially insecure.",
"default": false
},
"code_timeout_mins": {
"type": "number",
"description": "Number of minutes to allow code execution. -1 for no timeout",
"default": -1
},
"task": {
"type": "object",
"description": "The task object to give the agent on start. If null, the agent will not have a task.",
"default": null
}
}

4
src/models/groq.js
View file

@ -62,8 +62,8 @@ export class GroqCloudAPI {
}
log(JSON.stringify([{ role: "system", content: systemMessage }].concat(turns)), responseText);
// Original cleaning of <think> tags for the *returned* response (not affecting log)
responseText = responseText.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
return responseText;
res = responseText.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
return res;
} catch(err) {
if (err.message.includes("content must be a string")) {
res = "Vision is only supported by certain models.";

15
src/models/prompter.js
View file

@ -4,7 +4,7 @@ import { getCommandDocs } from '../agent/commands/index.js';
import { SkillLibrary } from "../agent/library/skill_library.js";
import { stringifyTurns } from '../utils/text.js';
import { getCommand } from '../agent/commands/index.js';
import settings from '../../settings.js';
import settings from '../agent/settings.js';
import { Gemini } from './gemini.js';
import { GPT } from './gpt.js';
@ -30,11 +30,18 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
export class Prompter {
constructor(agent, fp) {
constructor(agent, profile) {
this.agent = agent;
this.profile = JSON.parse(readFileSync(fp, 'utf8'));
this.profile = profile;
let default_profile = JSON.parse(readFileSync('./profiles/defaults/_default.json', 'utf8'));
let base_fp = settings.base_profile;
let base_fp = '';
if (settings.base_profile.includes('survival')) {
base_fp = './profiles/defaults/survival.json';
} else if (settings.base_profile.includes('creative')) {
base_fp = './profiles/defaults/creative.json';
} else if (settings.base_profile.includes('god_mode')) {
base_fp = './profiles/defaults/god_mode.json';
}
let base_profile = JSON.parse(readFileSync(base_fp, 'utf8'));
// first use defaults to fill in missing values in the base profile

1
src/models/vllm.js
View file

@ -48,6 +48,7 @@ export class VLLM {
try {
console.log('Awaiting openai api response...')
// console.log('Messages:', messages);
// todo set max_tokens, temperature, top_p, etc. in pack
let completion = await this.vllm.chat.completions.create(pack);
if (completion.choices[0].finish_reason == 'length')
throw new Error('Context length exceeded');

25
src/process/agent_process.js
View file

@ -1,23 +1,24 @@
import { spawn } from 'child_process';
import { mainProxy } from './main_proxy.js';
import { logoutAgent } from '../mindcraft/mindserver.js';
export class AgentProcess {
start(profile, load_memory=false, init_message=null, count_id=0, task_path=null, task_id=null) {
this.profile = profile;
constructor(name, port) {
this.name = name;
this.port = port;
}
start(load_memory=false, init_message=null, count_id=0) {
this.count_id = count_id;
this.running = true;
let args = ['src/process/init_agent.js', this.name];
args.push('-p', profile);
args.push('-n', this.name);
args.push('-c', count_id);
if (load_memory)
args.push('-l', load_memory);
if (init_message)
args.push('-m', init_message);
if (task_path)
args.push('-t', task_path);
if (task_id)
args.push('-i', task_id);
args.push('-p', this.port);
const agentProcess = spawn('node', args, {
stdio: 'inherit',
@ -28,7 +29,7 @@ export class AgentProcess {
agentProcess.on('exit', (code, signal) => {
console.log(`Agent process exited with code ${code} and signal ${signal}`);
this.running = false;
mainProxy.logoutAgent(this.name);
logoutAgent(this.name);
if (code > 1) {
console.log(`Ending task`);
@ -38,11 +39,11 @@ export class AgentProcess {
if (code !== 0 && signal !== 'SIGINT') {
// agent must run for at least 10 seconds before restarting
if (Date.now() - last_restart < 10000) {
console.error(`Agent process ${profile} exited too quickly and will not be restarted.`);
console.error(`Agent process exited too quickly and will not be restarted.`);
return;
}
console.log('Restarting agent...');
this.start(profile, true, 'Agent process restarted.', count_id, task_path, task_id);
this.start(true, 'Agent process restarted.', count_id, this.port);
last_restart = Date.now();
}
});
@ -61,7 +62,7 @@ export class AgentProcess {
continue() {
if (!this.running) {
this.start(this.profile, true, 'Agent process restarted.', this.count_id);
this.start(true, 'Agent process restarted.', this.count_id);
}
}
}

45
src/process/init_agent.js
View file

@ -1,27 +1,18 @@
import { Agent } from '../agent/agent.js';
import { serverProxy } from '../agent/mindserver_proxy.js';
import yargs from 'yargs';
// Add global unhandled rejection handler
process.on('unhandledRejection', (reason, promise) => {
console.error('Unhandled Rejection at:', {
promise: promise,
reason: reason,
stack: reason?.stack || 'No stack trace'
});
process.exit(1);
});
const args = process.argv.slice(2);
if (args.length < 1) {
console.log('Usage: node init_agent.js <agent_name> [profile] [load_memory] [init_message]');
console.log('Usage: node init_agent.js -n <agent_name> -p <port> -l <load_memory> -m <init_message> -c <count_id>');
process.exit(1);
}
const argv = yargs(args)
.option('profile', {
alias: 'p',
.option('name', {
alias: 'n',
type: 'string',
description: 'profile filepath to use for agent'
description: 'name of agent'
})
.option('load_memory', {
alias: 'l',
@ -33,29 +24,27 @@ const argv = yargs(args)
type: 'string',
description: 'automatically prompt the agent on startup'
})
.option('task_path', {
alias: 't',
type: 'string',
description: 'task filepath to use for agent'
})
.option('task_id', {
alias: 'i',
type: 'string',
description: 'task ID to execute'
})
.option('count_id', {
alias: 'c',
type: 'number',
default: 0,
description: 'identifying count for multi-agent scenarios',
}).argv;
})
.option('port', {
alias: 'p',
type: 'number',
description: 'port of mindserver'
})
.argv;
// Wrap agent start in async IIFE with proper error handling
(async () => {
try {
console.log('Starting agent with profile:', argv.profile);
console.log('Connecting to MindServer');
await serverProxy.connect(argv.name, argv.port);
console.log('Starting agent');
const agent = new Agent();
await agent.start(argv.profile, argv.load_memory, argv.init_message, argv.count_id, argv.task_path, argv.task_id);
serverProxy.setAgent(agent);
await agent.start(argv.load_memory, argv.init_message, argv.count_id);
} catch (error) {
console.error('Failed to start agent process:');
console.error(error.message);

64
src/process/main_proxy.js
View file

@ -1,64 +0,0 @@
import { io } from 'socket.io-client';
import settings from '../../settings.js';
// Singleton mindserver proxy for the main process
class MainProxy {
constructor() {
if (MainProxy.instance) {
return MainProxy.instance;
}
this.socket = null;
this.connected = false;
this.agent_processes = {};
MainProxy.instance = this;
}
connect() {
if (this.connected) return;
this.socket = io(`http://${settings.mindserver_host}:${settings.mindserver_port}`);
this.connected = true;
this.socket.on('stop-agent', (agentName) => {
if (this.agent_processes[agentName]) {
this.agent_processes[agentName].stop();
}
});
this.socket.on('start-agent', (agentName) => {
if (this.agent_processes[agentName]) {
this.agent_processes[agentName].continue();
}
});
this.socket.on('register-agents-success', () => {
console.log('Agents registered');
});
this.socket.on('shutdown', () => {
console.log('Shutting down');
for (let agentName in this.agent_processes) {
this.agent_processes[agentName].stop();
}
setTimeout(() => {
process.exit(0);
}, 2000);
});
}
addAgent(agent) {
this.agent_processes.push(agent);
}
logoutAgent(agentName) {
this.socket.emit('logout-agent', agentName);
}
registerAgent(name, process) {
this.socket.emit('register-agents', [name]);
this.agent_processes[name] = process;
}
}
export const mainProxy = new MainProxy();

166
src/server/mind_server.js
View file

@ -1,166 +0,0 @@
import { Server } from 'socket.io';
import express from 'express';
import http from 'http';
import path from 'path';
import { fileURLToPath } from 'url';
// Module-level variables
let io;
let server;
const registeredAgents = new Set();
const inGameAgents = {};
const agentManagers = {}; // socket for main process that registers/controls agents
// Initialize the server
export function createMindServer(port = 8080) {
const app = express();
server = http.createServer(app);
io = new Server(server);
// Serve static files
const __dirname = path.dirname(fileURLToPath(import.meta.url));
app.use(express.static(path.join(__dirname, 'public')));
// Socket.io connection handling
io.on('connection', (socket) => {
let curAgentName = null;
console.log('Client connected');
agentsUpdate(socket);
socket.on('register-agents', (agentNames) => {
console.log(`Registering agents: ${agentNames}`);
agentNames.forEach(name => registeredAgents.add(name));
for (let name of agentNames) {
agentManagers[name] = socket;
}
socket.emit('register-agents-success');
agentsUpdate();
});
socket.on('login-agent', (agentName) => {
if (curAgentName && curAgentName !== agentName) {
console.warn(`Agent ${agentName} already logged in as ${curAgentName}`);
return;
}
if (registeredAgents.has(agentName)) {
curAgentName = agentName;
inGameAgents[agentName] = socket;
agentsUpdate();
} else {
console.warn(`Agent ${agentName} not registered`);
}
});
socket.on('logout-agent', (agentName) => {
if (inGameAgents[agentName]) {
delete inGameAgents[agentName];
agentsUpdate();
}
});
socket.on('disconnect', () => {
console.log('Client disconnected');
if (inGameAgents[curAgentName]) {
delete inGameAgents[curAgentName];
agentsUpdate();
}
});
socket.on('chat-message', (agentName, json) => {
if (!inGameAgents[agentName]) {
console.warn(`Agent ${agentName} tried to send a message but is not logged in`);
return;
}
console.log(`${curAgentName} sending message to ${agentName}: ${json.message}`);
inGameAgents[agentName].emit('chat-message', curAgentName, json);
});
socket.on('restart-agent', (agentName) => {
console.log(`Restarting agent: ${agentName}`);
inGameAgents[agentName].emit('restart-agent');
});
socket.on('stop-agent', (agentName) => {
let manager = agentManagers[agentName];
if (manager) {
manager.emit('stop-agent', agentName);
}
else {
console.warn(`Stopping unregisterd agent ${agentName}`);
}
});
socket.on('start-agent', (agentName) => {
let manager = agentManagers[agentName];
if (manager) {
manager.emit('start-agent', agentName);
}
else {
console.warn(`Starting unregisterd agent ${agentName}`);
}
});
socket.on('stop-all-agents', () => {
console.log('Killing all agents');
stopAllAgents();
});
socket.on('shutdown', () => {
console.log('Shutting down');
for (let manager of Object.values(agentManagers)) {
manager.emit('shutdown');
}
setTimeout(() => {
process.exit(0);
}, 2000);
});
socket.on('send-message', (agentName, message) => {
if (!inGameAgents[agentName]) {
console.warn(`Agent ${agentName} not logged in, cannot send message via MindServer.`);
return
}
try {
console.log(`Sending message to agent ${agentName}: ${message}`);
inGameAgents[agentName].emit('send-message', agentName, message)
} catch (error) {
console.error('Error: ', error);
}
});
});
server.listen(port, 'localhost', () => {
console.log(`MindServer running on port ${port}`);
});
return server;
}
function agentsUpdate(socket) {
if (!socket) {
socket = io;
}
let agents = [];
registeredAgents.forEach(name => {
agents.push({name, in_game: !!inGameAgents[name]});
});
socket.emit('agents-update', agents);
}
function stopAllAgents() {
for (const agentName in inGameAgents) {
let manager = agentManagers[agentName];
if (manager) {
manager.emit('stop-agent', agentName);
}
}
}
// Optional: export these if you need access to them from other files
export const getIO = () => io;
export const getServer = () => server;
export const getConnectedAgents = () => connectedAgents;
export function getAllInGameAgentNames() {
return Object.keys(inGameAgents);
}

120
src/server/public/index.html
View file

@ -1,120 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Mindcraft</title>
<script src="/socket.io/socket.io.js"></script>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
background: #1a1a1a;
color: #e0e0e0;
}
#agents {
background: #2d2d2d;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.2);
}
h1 {
color: #ffffff;
}
.agent {
margin: 10px 0;
padding: 10px;
background: #363636;
border-radius: 4px;
display: flex;
justify-content: space-between;
align-items: center;
}
.restart-btn, .start-btn, .stop-btn {
color: white;
border: none;
padding: 5px 10px;
border-radius: 4px;
cursor: pointer;
margin-left: 5px;
}
.restart-btn {
background: #4CAF50;
}
.start-btn {
background: #2196F3;
}
.stop-btn {
background: #f44336;
}
.restart-btn:hover { background: #45a049; }
.start-btn:hover { background: #1976D2; }
.stop-btn:hover { background: #d32f2f; }
.status-icon {
font-size: 12px;
margin-right: 8px;
}
.status-icon.online {
color: #4CAF50;
}
.status-icon.offline {
color: #f44336;
}
</style>
</head>
<body>
<h1>Mindcraft</h1>
<div id="agents"></div>
<script>
const socket = io();
const agentsDiv = document.getElementById('agents');
socket.on('agents-update', (agents) => {
agentsDiv.innerHTML = agents.length ?
agents.map(agent => `
<div class="agent">
<span>
<span class="status-icon ${agent.in_game ? 'online' : 'offline'}">●</span>
${agent.name}
</span>
<div>
${agent.in_game ? `
<button class="stop-btn" onclick="stopAgent('${agent.name}')">Stop</button>
<button class="restart-btn" onclick="restartAgent('${agent.name}')">Restart</button>
<input type="text" id="messageInput" placeholder="Enter a message or command..."></input><button class="start-btn" onclick="sendMessage('${agent.name}', document.getElementById('messageInput').value)">Send</button>
` : `
<button class="start-btn" onclick="startAgent('${agent.name}')">Start</button>
`}
</div>
</div>
`).join('') +
`<button class="stop-btn" onclick="killAllAgents()">Stop All</button>
<button class="stop-btn" onclick="shutdown()">Shutdown</button>` :
'<div class="agent">No agents connected</div>';
});
function restartAgent(agentName) {
socket.emit('restart-agent', agentName);
}
function startAgent(agentName) {
socket.emit('start-agent', agentName);
}
function stopAgent(agentName) {
socket.emit('stop-agent', agentName);
}
function killAllAgents() {
socket.emit('stop-all-agents');
}
function shutdown() {
socket.emit('shutdown');
}
function sendMessage(agentName, message) {
socket.emit('send-message', agentName, message)
}
</script>
</body>
</html>

12
src/utils/mcdata.js
View file

@ -1,5 +1,5 @@
import minecraftData from 'minecraft-data';
import settings from '../../settings.js';
import settings from '../agent/settings.js';
import { createBot } from 'mineflayer';
import prismarine_items from 'prismarine-item';
import { pathfinder } from 'mineflayer-pathfinder';
@ -8,10 +8,9 @@ import { plugin as collectblock } from 'mineflayer-collectblock';
import { plugin as autoEat } from 'mineflayer-auto-eat';
import plugin from 'mineflayer-armor-manager';
const armorManager = plugin;
const mc_version = settings.minecraft_version;
const mcdata = minecraftData(mc_version);
const Item = prismarine_items(mc_version);
let mc_version = null;
let mcdata = null;
let Item = null;
/**
* @typedef {string} ItemName
@ -54,6 +53,9 @@ export const WOOL_COLORS = [
export function initBot(username) {
mc_version = settings.minecraft_version;
mcdata = minecraftData(mc_version);
Item = prismarine_items(mc_version);
let bot = createBot({
username: username,

10
src/utils/translator.js
View file

@ -1,10 +1,11 @@
import translate from 'google-translate-api-x';
import settings from '../../settings.js';
import settings from '../agent/settings.js';
const preferred_lang = String(settings.language).toLowerCase();
export async function handleTranslation(message) {
if (preferred_lang === 'en' || preferred_lang === 'english')
let preferred_lang = String(settings.language).toLowerCase();
if (!preferred_lang || preferred_lang === 'en' || preferred_lang === 'english')
return message;
try {
const translation = await translate(message, { to: preferred_lang });
@ -16,7 +17,8 @@ export async function handleTranslation(message) {
}
export async function handleEnglishTranslation(message) {
if (preferred_lang === 'en' || preferred_lang === 'english')
let preferred_lang = String(settings.language).toLowerCase();
if (!preferred_lang || preferred_lang === 'en' || preferred_lang === 'english')
return message;
try {
const translation = await translate(message, { to: 'english' });

0
tasks/construction_tasks/church_three_agents.pdf → tasks/construction_tasks/church_blueprint.pdf
View file

2
tasks/construction_tasks/human_ai/1_agent_1_human.json
View file

@ -2351,7 +2351,7 @@
}
},
"usernames": [
"izzycw"
"erringnine"
]
}
}

38
tasks/cooking_tasks/human_ai/1_agent_1_human.json Normal file
View file

@ -0,0 +1,38 @@
{
"multiagent_cooking_bread_golden_apple": {
"type": "cooking",
"recipes": {
"bread": [
"Step 1: Go to the farm and collect 3 wheat.",
"Step 2: Go to the crafting table and use the wheat to craft bread."
],
"golden_apple": [
"Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.",
"Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple."
]
},
"agent_count": 1,
"human_count": 1,
"target": {
"bread": 1,
"golden_apple": 1
},
"initial_inventory": {
"0": {
"gold_ingot": 4,
"apple": 1
},
"1": {
"gold_ingot": 4
}
},
"goal": {
"0": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n",
"1": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n"
},
"conversation": "Let's collaborate to make bread, golden_apple, ",
"usernames": [
"izzycw"
]
}
}

31
tasks/crafting_tasks/human_ai/1_agent_1_human.json Normal file
View file

@ -0,0 +1,31 @@
{
"multiagent_crafting_pink_wool_full_plan__depth_0": {
"goal": "Collaborate with other agents to craft an pink_wool",
"conversation": "Let's work together to craft an pink_wool.",
"initial_inventory": {
"0": {
"pink_dye": 1
},
"1": {
"black_wool": 1
}
},
"agent_count": 1,
"target": "pink_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 0,
"timeout": 300,
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"human_count": 1,
"usernames": [
"izzycw"
],
"requires_ctable": false
}
}

0
tasks/crafting_tasks/human_ai_tasks/1_agent_1_human_copy.json → tasks/crafting_tasks/human_ai/1_agent_1_human_copy.json
View file

0
tasks/crafting_tasks/human_ai_tasks/3_agent_1_human.json → tasks/crafting_tasks/human_ai/3_agent_1_human.json
View file

793
tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json
View file

@ -1,793 +0,0 @@
{
"multiagent_crafting_pink_wool_full_plan__depth_0": {
"goal": "Collaborate with other agents to craft an pink_wool",
"conversation": "Let's work together to craft an pink_wool.",
"initial_inventory": {
"0": {
"pink_dye": 1
},
"1": {
"black_wool": 1
}
},
"agent_count": 1,
"target": "pink_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 0,
"timeout": 300,
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"human_count": 1,
"usernames": [
"izzycw"
],
"requires_ctable": false
},
"multiagent_crafting_lime_wool_partial_plan__depth_0": {
"goal": "Collaborate with other agents to craft an lime_wool",
"conversation": "Let's work together to craft an lime_wool.",
"initial_inventory": {
"0": {
"lime_dye": 1
},
"1": {
"black_wool": 1
}
},
"agent_count": 1,
"target": "lime_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"blocked_actions": {
"0": [],
"1": [
"!getCraftingPlan"
]
},
"human_count": 1,
"usernames": [
"izzycw"
],
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_purple_banner_full_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft an purple_banner",
"conversation": "Let's work together to craft an purple_banner.",
"initial_inventory": {
"0": {
"purple_wool": 4,
"stick": 1
},
"1": {
"purple_wool": 3,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "purple_banner",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_soul_campfire_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft an soul_campfire",
"conversation": "Let's work together to craft an soul_campfire.",
"initial_inventory": {
"0": {
"oak_planks": 2,
"soul_sand": 1,
"dark_oak_log": 2
},
"1": {
"oak_planks": 1,
"dark_oak_log": 1,
"crafting_table": 1
}
},
"agent_count": 2,
"target": "soul_campfire",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_bookshelf_full_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a bookshelf",
"conversation": "Let's work together to craft a bookshelf.",
"initial_inventory": {
"0": {
"oak_planks": 4,
"book": 2
},
"1": {
"oak_planks": 2,
"book": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "bookshelf",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_compass_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a compass",
"conversation": "Let's work together to craft a compass.",
"initial_inventory": {
"0": {
"iron_ingot": 2
},
"1": {
"iron_ingot": 2,
"redstone": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "compass",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_fishing_rod_full_plan_requires_ctable__depth_1": {
"goal": "Collaborate with other agents to craft a fishing_rod",
"conversation": "Let's work together to craft a fishing_rod.",
"initial_inventory": {
"0": {
"string": 1,
"oak_planks": 2
},
"1": {
"string": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "fishing_rod",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_cake_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a cake",
"conversation": "Let's work together to craft a cake.",
"initial_inventory": {
"0": {
"wheat": 2,
"sugar": 1,
"egg": 1
},
"1": {
"wheat": 1,
"milk_bucket": 2,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "cake",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_golden_carrot_full_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a golden_carrot",
"conversation": "Let's work together to craft a golden_carrot.",
"initial_inventory": {
"0": {
"gold_nugget": 5,
"carrot": 1
},
"1": {
"gold_nugget": 3,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "golden_carrot",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_map_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a map",
"conversation": "Let's work together to craft a map.",
"initial_inventory": {
"0": {
"paper": 5
},
"1": {
"paper": 3,
"compass": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "map",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_blue_wool_full_plan__depth_0": {
"goal": "Collaborate with other agents to craft blue_wool",
"conversation": "Let's work together to craft blue_wool.",
"initial_inventory": {
"0": {
"blue_dye": 1
},
"1": {
"white_wool": 1
}
},
"agent_count": 2,
"target": "blue_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_lime_wool_partial_plan__depth_2": {
"goal": "Collaborate with other agents to craft lime_wool",
"conversation": "Let's work together to craft lime_wool.",
"initial_inventory": {
"0": {
"green_dye": 1
},
"1": {
"white_wool": 1,
"bone_meal": 1
}
},
"agent_count": 2,
"target": "lime_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_magenta_wool_full_plan__depth_2": {
"goal": "Collaborate with other agents to craft magenta_wool",
"conversation": "Let's work together to craft magenta_wool.",
"initial_inventory": {
"0": {
"rose_red": 1,
"lapis_lazuli": 1
},
"1": {
"white_wool": 1,
"bone_meal": 1
}
},
"agent_count": 1,
"target": "magenta_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 2,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_chest_full_plan_requires_ctable__depth_1": {
"goal": "Collaborate with other agents to craft a chest",
"conversation": "Let's work together to craft a chest.",
"initial_inventory": {
"0": {
"oak_log": 1
},
"1": {
"oak_planks": 4,
"crafting_table": 1
}
},
"agent_count": 2,
"target": "chest",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_barrel_partial_plan_requires_ctable__depth_1": {
"goal": "Collaborate with other agents to craft a barrel",
"conversation": "Let's work together to craft a barrel.",
"initial_inventory": {
"0": {
"spruce_planks": 3,
"crafting_table": 1
},
"1": {
"spruce_planks": 3,
"wooden_slab": 1
}
},
"agent_count": 2,
"target": "barrel",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_lectern_full_plan_requires_ctable__depth_2": {
"goal": "Collaborate with other agents to craft a lectern",
"conversation": "Let's work together to craft a lectern.",
"initial_inventory": {
"0": {
"birch_slab": 5,
"crafting_table": 1
},
"1": {
"birch_log": 2,
"book": 3
}
},
"agent_count": 1,
"target": "lectern",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 2,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_clock_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft a clock",
"conversation": "Let's work together to craft a clock.",
"initial_inventory": {
"0": {
"gold_ingot": 2
},
"1": {
"gold_ingot": 2,
"redstone": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "clock",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_firework_rocket_partial_plan__depth_0": {
"goal": "Collaborate with other agents to craft firework_rocket",
"conversation": "Let's work together to craft firework_rocket.",
"initial_inventory": {
"0": {
"paper": 1
},
"1": {
"gunpowder": 3
}
},
"agent_count": 2,
"target": "firework_rocket",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_enchanting_table_partial_plan_requires_ctable__depth_0": {
"goal": "Collaborate with other agents to craft an enchanting_table",
"conversation": "Let's work together to craft an enchanting_table.",
"initial_inventory": {
"0": {
"diamond": 2,
"obsidian": 2,
"crafting_table": 1
},
"1": {
"obsidian": 2,
"book": 1
}
},
"agent_count": 2,
"target": "enchanting_table",
"number_of_target": 1,
"type": "techtree",
"max_depth": 0,
"depth": 0,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_jukebox_full_plan_requires_ctable__depth_1": {
"goal": "Collaborate with other agents to craft a jukebox",
"conversation": "Let's work together to craft a jukebox.",
"initial_inventory": {
"0": {
"diamond": 1
},
"1": {
"oak_log": 2,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "jukebox",
"number_of_target": 1,
"type": "techtree",
"max_depth": 1,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_light_gray_wool_full_plan__depth_1": {
"goal": "Collaborate with other agents to craft light_gray_wool",
"conversation": "Let's work together to craft light_gray_wool.",
"initial_inventory": {
"0": {
"black_dye": 1
},
"1": {
"white_wool": 1,
"white_dye": 2
}
},
"agent_count": 1,
"target": "light_gray_wool",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": false
},
"multiagent_crafting_blast_furnace_full_plan_requires_ctable__depth_1": {
"goal": "Collaborate with other agents to craft a blast_furnace",
"conversation": "Let's work together to craft a blast_furnace.",
"initial_inventory": {
"0": {
"iron_ingot": 5,
"smooth_stone": 3
},
"1": {
"cobblestone": 8,
"crafting_table": 1
}
},
"agent_count": 2,
"target": "blast_furnace",
"number_of_target": 1,
"type": "techtree",
"max_depth": 2,
"depth": 1,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_activator_rail_full_plan_requires_ctable__depth_2": {
"goal": "Collaborate with other agents to craft activator_rail",
"conversation": "Let's work together to craft activator_rail.",
"initial_inventory": {
"0": {
"iron_ingot": 3,
"oak_planks": 6
},
"1": {
"redstone": 1,
"iron_ingot": 3,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "activator_rail",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 2,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_campfire_partial_plan_requires_ctable__depth_2": {
"goal": "Collaborate with other agents to craft campfire",
"conversation": "Let's work together to craft campfire.",
"initial_inventory": {
"0": {
"oak_log": 8
},
"1": {
"coal": 1,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "campfire",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 2,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [
"!getCraftingPlan"
],
"1": []
},
"missing_items": [],
"requires_ctable": true
},
"multiagent_crafting_crossbow_full_plan_requires_ctable__depth_2": {
"goal": "Collaborate with other agents to craft a crossbow",
"conversation": "Let's work together to craft a crossbow.",
"initial_inventory": {
"0": {
"oak_planks": 8,
"iron_ingot": 2
},
"1": {
"string": 2,
"crafting_table": 1
}
},
"agent_count": 1,
"target": "crossbow",
"number_of_target": 1,
"type": "techtree",
"max_depth": 3,
"depth": 2,
"timeout": 300,
"human_count": 1,
"usernames": [
"izzycw"
],
"blocked_actions": {
"0": [],
"1": []
},
"missing_items": [],
"requires_ctable": true
}
}

5
tasks/evaluation_script.py
View file

@ -179,7 +179,8 @@ def check_folder_results(folder_path):
# Print summary
print("\n=== Evaluation Results ===")
print(f"Total tasks evaluated: {results['total']}")
print("\nEvaluating Tasks!")
print(f"Results so far: {results['total']}")
if "construction" not in folder_path:
print(f"Successful tasks: {results['successful']}")
@ -517,7 +518,7 @@ def make_ops(agent_names, session_name):
if agents_op:
print("Agents are operators! You are good to go :D")
else:
print("Agents are not operators! Something went wrong :(")
print("Agents are not operators! We will need to try making them operators again!")
make_ops(agent_names, session_name)
def check_agent_ops(agent_names, ops_file="ops.json"):

2
tasks/example_tasks.json
View file

@ -26,7 +26,7 @@
}
},
"type": "debug",
"timeout": 60
"timeout": 25
},
"debug_2_agent_timeout": {
"goal": "Just stand at a place and don't do anything",

34
tasks/running_human_ai.md Normal file
View file

@ -0,0 +1,34 @@
# Human AI Instructions
## Finishing Installation
Install the conda environment for running the experiments by executing this in your command line:
```
conda create --name mindcraft python=3.11
conda activate mindcraft
pip install -r requirements.txt
```
## Setting up the world
Setting up the world! Make sure your world has cheats enabled! You can do this on creation of your Minecraft world in the Minecraft console, or you can type ```/op @a``` in the chat or in the console of the world launched from the jar file.
## Construction
Press F3 to view the coordinates of the game. And pull up the file tasks/construction_tasks/church_blueprint.pdf
Run
```
python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/construction_profile.json --task_path tasks/construction_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1 --insecure_coding
```
## Crafting
```
python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/crafting_profile.json --task_path tasks/crafting_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1
```
## Cooking
```
python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/cooking_profile.json --task_path tasks/cooking_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1
```

69
viewer.html
View file

@ -1,69 +0,0 @@
<html>
<head>
<title>Viewer</title>
<style>
body {
margin: 0;
padding: 0;
}
#container {
display: flex;
flex-wrap: wrap;
width: 100%;
height: 100vh;
}
.iframe-wrapper {
border: none;
}
</style>
</head>
<body>
<div id="container">
<iframe data-port="3000" src="http://localhost:3000" class="iframe-wrapper"></iframe>
<iframe data-port="3001" src="http://localhost:3001" class="iframe-wrapper"></iframe>
<iframe data-port="3002" src="http://localhost:3002" class="iframe-wrapper"></iframe>
<iframe data-port="3003" src="http://localhost:3003" class="iframe-wrapper"></iframe>
</div>
<script>
function updateLayout() {
let width = window.innerWidth;
let height = window.innerHeight;
let iframes = document.querySelectorAll('.iframe-wrapper');
if (width > height) {
iframes.forEach(function(iframe) {
iframe.style.width = '50%';
iframe.style.height = '50%';
});
} else {
iframes.forEach(function(iframe) {
iframe.style.width = '100%';
iframe.style.height = '25%';
});
}
}
window.addEventListener('resize', updateLayout);
window.addEventListener('load', updateLayout);
let iframes = document.querySelectorAll('.iframe-wrapper');
iframes.forEach(function(iframe) {
let port = iframe.getAttribute('data-port');
let loaded = false;
function checkServer() {
fetch('http://localhost:' + port, { method: 'HEAD' })
.then(function(response) {
if (response.ok && !loaded) {
iframe.src = 'http://localhost:' + port;
}
})
.catch(function(error) {});
}
iframe.onload = function() {
loaded = true;
};
iframe.onerror = function() {
loaded = false;
};
setInterval(checkServer, 3000);
});
</script>
</body>
</html>