diff --git a/main.js b/main.js index 590348c..4402cb9 100644 --- a/main.js +++ b/main.js @@ -63,7 +63,7 @@ if (process.env.LOG_ALL) { settings.log_all_prompts = process.env.LOG_ALL; } -Mindcraft.init(false, settings.mindserver_port); +Mindcraft.init(false, settings.mindserver_port, settings.auto_open_ui); for (let profile of settings.profiles) { const profile_json = JSON.parse(readFileSync(profile, 'utf8')); diff --git a/package.json b/package.json index f7f141f..ac2dcc9 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "mineflayer-pvp": "^1.3.2", "node-canvas-webgl": "PrismarineJS/node-canvas-webgl", "npm": "^11.5.2", + "open": "^10.2.0", "openai": "^4.4.0", "prismarine-item": "^1.15.0", "prismarine-viewer": "^1.32.0", diff --git a/settings.js b/settings.js index e47ab0d..f882692 100644 --- a/settings.js +++ b/settings.js @@ -1,11 +1,12 @@ const settings = { "minecraft_version": "auto", // or specific version like "1.21.1" "host": "127.0.0.1", // or "localhost", "your.ip.address.here" - "port": 55916, + "port": 55916, // set to -1 to automatically scan for open ports "auth": "offline", // or "microsoft" // the mindserver manages all agents and hosts the UI "mindserver_port": 8080, + "auto_open_ui": true, // opens UI in browser on startup "base_profile": "assistant", // survival, assistant, creative, or god_mode "profiles": [ diff --git a/src/agent/library/full_state.js b/src/agent/library/full_state.js new file mode 100644 index 0000000..45a1fbe --- /dev/null +++ b/src/agent/library/full_state.js @@ -0,0 +1,89 @@ +import { + getPosition, + getBiomeName, + getNearbyPlayerNames, + getInventoryCounts, + getNearbyEntityTypes, + getBlockAtPosition, + getFirstBlockAboveHead +} from "./world.js"; +import convoManager from '../conversation.js'; + +export function getFullState(agent) { + const bot = agent.bot; + + const pos = getPosition(bot); + const position = { + x: Number(pos.x.toFixed(2)), + y: Number(pos.y.toFixed(2)), + z: Number(pos.z.toFixed(2)) + }; + + let weather = 'Clear'; + if (bot.thunderState > 0) weather = 'Thunderstorm'; + else if (bot.rainState > 0) weather = 'Rain'; + + let timeLabel = 'Night'; + if (bot.time.timeOfDay < 6000) timeLabel = 'Morning'; + else if (bot.time.timeOfDay < 12000) timeLabel = 'Afternoon'; + + const below = getBlockAtPosition(bot, 0, -1, 0).name; + const legs = getBlockAtPosition(bot, 0, 0, 0).name; + const head = getBlockAtPosition(bot, 0, 1, 0).name; + + let players = getNearbyPlayerNames(bot); + let bots = convoManager.getInGameAgents().filter(b => b !== agent.name); + players = players.filter(p => !bots.includes(p)); + + const helmet = bot.inventory.slots[5]; + const chestplate = bot.inventory.slots[6]; + const leggings = bot.inventory.slots[7]; + const boots = bot.inventory.slots[8]; + + const state = { + name: agent.name, + gameplay: { + position, + dimension: bot.game.dimension, + gamemode: bot.game.gameMode, + health: Math.round(bot.health), + hunger: Math.round(bot.food), + biome: getBiomeName(bot), + weather, + timeOfDay: bot.time.timeOfDay, + timeLabel + }, + action: { + current: agent.isIdle() ? 'Idle' : agent.actions.currentActionLabel, + isIdle: agent.isIdle() + }, + surroundings: { + below, + legs, + head, + firstBlockAboveHead: getFirstBlockAboveHead(bot, null, 32) + }, + inventory: { + counts: getInventoryCounts(bot), + stacksUsed: bot.inventory.items().length, + totalSlots: bot.inventory.slots.length, + equipment: { + helmet: helmet ? helmet.name : null, + chestplate: chestplate ? chestplate.name : null, + leggings: leggings ? leggings.name : null, + boots: boots ? boots.name : null, + mainHand: bot.heldItem ? bot.heldItem.name : null + } + }, + nearby: { + humanPlayers: players, + botPlayers: bots, + entityTypes: getNearbyEntityTypes(bot).filter(t => t !== 'player' && t !== 'item'), + }, + modes: { + summary: bot.modes.getMiniDocs() + } + }; + + return state; +} \ No newline at end of file diff --git a/src/agent/mindserver_proxy.js b/src/agent/mindserver_proxy.js index 1426098..a64fe94 100644 --- a/src/agent/mindserver_proxy.js +++ b/src/agent/mindserver_proxy.js @@ -1,6 +1,7 @@ import { io } from 'socket.io-client'; import convoManager from './conversation.js'; import { setSettings } from './settings.js'; +import { getFullState } from './library/full_state.js'; // agent's individual connection to the mindserver // always connect to localhost @@ -43,7 +44,7 @@ class MindServerProxy { convoManager.receiveFromBot(agentName, json); }); - this.socket.on('agents-update', (agents) => { + this.socket.on('agents-status', (agents) => { this.agents = agents; convoManager.updateAgents(agents); if (this.agent?.task) { @@ -57,14 +58,24 @@ class MindServerProxy { this.agent.cleanKill(); }); - this.socket.on('send-message', (agentName, message) => { + this.socket.on('send-message', (data) => { try { - this.agent.respondFunc("NO USERNAME", message); + this.agent.respondFunc(data.from, data.message); } catch (error) { console.error('Error: ', JSON.stringify(error, Object.getOwnPropertyNames(error))); } }); + this.socket.on('get-full-state', (callback) => { + try { + const state = getFullState(this.agent); + callback(state); + } catch (error) { + console.error('Error getting full state:', error); + callback(null); + } + }); + // Request settings and wait for response await new Promise((resolve, reject) => { const timeout = setTimeout(() => { @@ -77,6 +88,7 @@ class MindServerProxy { return reject(new Error(response.error)); } setSettings(response.settings); + this.socket.emit('connect-agent-process', name); resolve(); }); }); diff --git a/src/mindcraft/mindcraft.js b/src/mindcraft/mindcraft.js index 57c4dfe..a860b87 100644 --- a/src/mindcraft/mindcraft.js +++ b/src/mindcraft/mindcraft.js @@ -1,6 +1,7 @@ -import { createMindServer, registerAgent } from './mindserver.js'; +import { createMindServer, registerAgent, numStateListeners } from './mindserver.js'; import { AgentProcess } from '../process/agent_process.js'; import { getServer } from './mcserver.js'; +import open from 'open'; let mindserver; let connected = false; @@ -8,7 +9,7 @@ let agent_processes = {}; let agent_count = 0; let port = 8080; -export async function init(host_public=false, port=8080) { +export async function init(host_public=false, port=8080, auto_open_ui=true) { if (connected) { console.error('Already initiliazed!'); return; @@ -16,28 +17,53 @@ export async function init(host_public=false, port=8080) { mindserver = createMindServer(host_public, port); port = port; connected = true; + if (auto_open_ui) { + setTimeout(() => { + // check if browser listener is already open + if (numStateListeners() === 0) { + open('http://localhost:'+port); + } + }, 3000); + } } export async function createAgent(settings) { if (!settings.profile.name) { console.error('Agent name is required in profile'); - return; + return { + success: false, + error: 'Agent name is required in profile' + }; } settings = JSON.parse(JSON.stringify(settings)); let agent_name = settings.profile.name; - registerAgent(settings); + const viewer_port = 3000 + agent_count; + registerAgent(settings, viewer_port); let load_memory = settings.load_memory || false; let init_message = settings.init_message || null; - const server = await getServer(settings.host, settings.port, settings.minecraft_version); - settings.host = server.host; - settings.port = server.port; - settings.minecraft_version = server.version; + try { + const server = await getServer(settings.host, settings.port, settings.minecraft_version); + settings.host = server.host; + settings.port = server.port; + settings.minecraft_version = server.version; - const agentProcess = new AgentProcess(agent_name, port); - agentProcess.start(load_memory, init_message, agent_count); - agent_count++; - agent_processes[settings.profile.name] = agentProcess; + const agentProcess = new AgentProcess(agent_name, port); + agentProcess.start(load_memory, init_message, agent_count); + agent_count++; + agent_processes[settings.profile.name] = agentProcess; + } catch (error) { + console.error(`Error creating agent ${agent_name}:`, error); + destroyAgent(agent_name); + return { + success: false, + error: error.message + }; + } + return { + success: true, + error: null + }; } export function getAgentProcess(agentName) { @@ -59,6 +85,13 @@ export function stopAgent(agentName) { } } +export function destroyAgent(agentName) { + if (agent_processes[agentName]) { + agent_processes[agentName].stop(); + delete agent_processes[agentName]; + } +} + export function shutdown() { console.log('Shutting down'); for (let agentName in agent_processes) { diff --git a/src/mindcraft/mindserver.js b/src/mindcraft/mindserver.js index acd0775..1397553 100644 --- a/src/mindcraft/mindserver.js +++ b/src/mindcraft/mindserver.js @@ -15,27 +15,32 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); let io; let server; const agent_connections = {}; +const agent_listeners = []; const settings_spec = JSON.parse(readFileSync(path.join(__dirname, 'public/settings_spec.json'), 'utf8')); class AgentConnection { - constructor(settings) { + constructor(settings, viewer_port) { this.socket = null; this.settings = settings; this.in_game = false; + this.full_state = null; + this.viewer_port = viewer_port; + } + setSettings(settings) { + this.settings = settings; } - } -export function registerAgent(settings) { - let agentConnection = new AgentConnection(settings); +export function registerAgent(settings, viewer_port) { + let agentConnection = new AgentConnection(settings, viewer_port); agent_connections[settings.profile.name] = agentConnection; } export function logoutAgent(agentName) { if (agent_connections[agentName]) { agent_connections[agentName].in_game = false; - agentsUpdate(); + agentsStatusUpdate(); } } @@ -54,9 +59,9 @@ export function createMindServer(host_public = false, port = 8080) { let curAgentName = null; console.log('Client connected'); - agentsUpdate(socket); + agentsStatusUpdate(socket); - socket.on('create-agent', (settings, callback) => { + socket.on('create-agent', async (settings, callback) => { console.log('API create agent...'); for (let key in settings_spec) { if (!(key in settings)) { @@ -79,8 +84,14 @@ export function createMindServer(host_public = false, port = 8080) { callback({ success: false, error: 'Agent already exists' }); return; } - mindcraft.createAgent(settings); - callback({ success: true }); + let returned = await mindcraft.createAgent(settings); + callback({ success: returned.success, error: returned.error }); + let name = settings.profile.name; + if (!returned.success && agent_connections[name]) { + mindcraft.destroyAgent(name); + delete agent_connections[name]; + } + agentsStatusUpdate(); } else { console.error('Agent name is required in profile'); @@ -96,12 +107,19 @@ export function createMindServer(host_public = false, port = 8080) { } }); + socket.on('connect-agent-process', (agentName) => { + if (agent_connections[agentName]) { + agent_connections[agentName].socket = socket; + agentsStatusUpdate(); + } + }); + socket.on('login-agent', (agentName) => { if (agent_connections[agentName]) { agent_connections[agentName].socket = socket; agent_connections[agentName].in_game = true; curAgentName = agentName; - agentsUpdate(); + agentsStatusUpdate(); } else { console.warn(`Unregistered agent ${agentName} tried to login`); @@ -112,7 +130,11 @@ export function createMindServer(host_public = false, port = 8080) { if (agent_connections[curAgentName]) { console.log(`Agent ${curAgentName} disconnected`); agent_connections[curAgentName].in_game = false; - agentsUpdate(); + agent_connections[curAgentName].socket = null; + agentsStatusUpdate(); + } + if (agent_listeners.includes(socket)) { + removeListener(socket); } }); @@ -125,6 +147,14 @@ export function createMindServer(host_public = false, port = 8080) { agent_connections[agentName].socket.emit('chat-message', curAgentName, json); }); + socket.on('set-agent-settings', (agentName, settings) => { + const agent = agent_connections[agentName]; + if (agent) { + agent.setSettings(settings); + agent.socket.emit('restart-agent'); + } + }); + socket.on('restart-agent', (agentName) => { console.log(`Restarting agent: ${agentName}`); agent_connections[agentName].socket.emit('restart-agent'); @@ -138,6 +168,14 @@ export function createMindServer(host_public = false, port = 8080) { mindcraft.startAgent(agentName); }); + socket.on('destroy-agent', (agentName) => { + if (agent_connections[agentName]) { + mindcraft.destroyAgent(agentName); + delete agent_connections[agentName]; + } + agentsStatusUpdate(); + }); + socket.on('stop-all-agents', () => { console.log('Killing all agents'); for (let agentName in agent_connections) { @@ -158,14 +196,13 @@ export function createMindServer(host_public = false, port = 8080) { }); - socket.on('send-message', (agentName, message) => { + socket.on('send-message', (agentName, data) => { if (!agent_connections[agentName]) { console.warn(`Agent ${agentName} not in game, cannot send message via MindServer.`); return } try { - console.log(`Sending message to agent ${agentName}: ${message}`); - agent_connections[agentName].socket.emit('send-message', agentName, message) + agent_connections[agentName].socket.emit('send-message', data) } catch (error) { console.error('Error: ', error); } @@ -174,6 +211,10 @@ export function createMindServer(host_public = false, port = 8080) { socket.on('bot-output', (agentName, message) => { io.emit('bot-output', agentName, message); }); + + socket.on('listen-to-agents', () => { + addListener(socket); + }); }); let host = host_public ? '0.0.0.0' : 'localhost'; @@ -184,17 +225,59 @@ export function createMindServer(host_public = false, port = 8080) { return server; } -function agentsUpdate(socket) { +function agentsStatusUpdate(socket) { if (!socket) { socket = io; } let agents = []; for (let agentName in agent_connections) { - agents.push({name: agentName, in_game: agent_connections[agentName].in_game}); + const conn = agent_connections[agentName]; + agents.push({ + name: agentName, + in_game: conn.in_game, + viewerPort: conn.viewer_port, + socket_connected: !!conn.socket + }); }; - socket.emit('agents-update', agents); + socket.emit('agents-status', agents); +} + + +let listenerInterval = null; +function addListener(listener_socket) { + agent_listeners.push(listener_socket); + if (agent_listeners.length === 1) { + listenerInterval = setInterval(async () => { + const states = {}; + for (let agentName in agent_connections) { + let agent = agent_connections[agentName]; + if (agent.in_game) { + try { + const state = await new Promise((resolve) => { + agent.socket.emit('get-full-state', (s) => resolve(s)); + }); + states[agentName] = state; + } catch (e) { + states[agentName] = { error: String(e) }; + } + } + } + for (let listener of agent_listeners) { + listener.emit('state-update', states); + } + }, 1000); + } +} + +function removeListener(listener_socket) { + agent_listeners.splice(agent_listeners.indexOf(listener_socket), 1); + if (agent_listeners.length === 0) { + clearInterval(listenerInterval); + listenerInterval = null; + } } // Optional: export these if you need access to them from other files export const getIO = () => io; export const getServer = () => server; +export const numStateListeners = () => agent_listeners.length; \ No newline at end of file diff --git a/src/mindcraft/public/index.html b/src/mindcraft/public/index.html index d9690a2..8a29c89 100644 --- a/src/mindcraft/public/index.html +++ b/src/mindcraft/public/index.html @@ -26,7 +26,7 @@ border-radius: 4px; display: flex; flex-direction: column; - align-items: flex-start; + align-items: stretch; } .restart-btn, .start-btn, .stop-btn { color: white; @@ -48,6 +48,17 @@ .restart-btn:hover { background: #45a049; } .start-btn:hover { background: #1976D2; } .stop-btn:hover { background: #d32f2f; } + .gear-btn { + background: #505050; + color: #fff; + border: none; + border-radius: 4px; + padding: 2px 6px; + cursor: pointer; + margin-left: 6px; + font-size: 0.9em; + } + .gear-btn:hover { background: #5a5a5a; } .status-icon { font-size: 12px; margin-right: 8px; @@ -96,18 +107,26 @@ .setting-wrapper input[type="checkbox"] { transform: scale(1.2); } + .agent-view-container { + width: 100%; + height: 100%; + aspect-ratio: 4/3; + overflow: hidden; + } .agent-viewer { - width: 200px; - height: 150px; + width: 100%; + height: 100%; border: none; - margin-left: 10px; + display: block; } .last-message { font-style: italic; color: #aaa; margin-top: 5px; - white-space: pre-wrap; - word-break: break-word; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + width: 100%; } .start-btn:disabled { opacity: 0.4; @@ -118,22 +137,243 @@ display: flex; justify-content: flex-start; } + .agent-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); + gap: 8px; + width: 100%; + align-items: start; + } + .agent-grid .cell { + background: #3a3a3a; + padding: 6px 8px; + border-radius: 4px; + } + .agent-grid .cell.title { + background: transparent; + padding: 0; + } + .agent-inventory { + margin-top: 8px; + background: #2f2f2f; + border-radius: 6px; + padding: 8px; + } + .agent-inventory h3 { margin: 0 0 6px 0; font-size: 1em; } + .inventory-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 6px; + } + .agent-details-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 6px; + width: 100%; + } + .controls-row { + margin-top: 8px; + display: grid; + grid-template-columns: auto minmax(100px, 1fr) repeat(5, auto); + gap: 8px; + align-items: center; + } + .msg-input { + width: calc(100% - 8px); + background: #262626; + border: 1px solid #555; + color: #e0e0e0; + border-radius: 4px; + padding: 4px 6px; + } + .neutral-btn { + color: white; + border: none; + padding: 5px 10px; + border-radius: 4px; + cursor: pointer; + margin-left: 5px; + background: #505050; + } + .neutral-btn:hover { background: #5a5a5a; } + .neutral-btn:disabled { + background: #383838; + color: #666; + cursor: not-allowed; + } + .msg-input:disabled { + background: #1a1a1a; + color: #666; + cursor: not-allowed; + } + .page-footer { + position: fixed; + bottom: 0; + left: 0; + right: 0; + background: #2d2d2d; + padding: 16px; + display: flex; + gap: 12px; + justify-content: space-between; + box-shadow: 0 -2px 10px rgba(0,0,0,0.2); + } + body { + padding-bottom: 80px; /* Make room for footer */ + } + .agent-stats-row { + display: grid; + grid-template-columns: repeat(5, 1fr); + gap: 8px; + margin: 6px 0; + font-size: 0.9em; + color: #cccccc; + width: 100%; + box-sizing: border-box; + } + .agent-stats-row .stat { + background: #3a3a3a; + padding: 6px 8px; + border-radius: 4px; + } + .status-badge { + font-size: 0.75em; + margin-left: 8px; + padding: 2px 6px; + border-radius: 4px; + background: #3a3a3a; + color: #cccccc; + text-transform: lowercase; + } + .status-badge.online { color: #4CAF50; } + .status-badge.offline { color: #f44336; } + .title-row { + display: flex; + align-items: center; + gap: 10px; + } + .title-left { + display: flex; + align-items: center; + gap: 8px; + } + .title-spacer { flex: 1; } + /* Modal styles */ + .modal-backdrop { + position: fixed; + top: 0; left: 0; right: 0; bottom: 0; + background: rgba(0,0,0,0.5); + display: none; + align-items: center; + justify-content: center; + z-index: 1000; + } + .modal { + background: #2d2d2d; + border-radius: 8px; + width: 80vw; + height: 80vh; + display: flex; + flex-direction: column; + overflow: hidden; + box-shadow: 0 4px 12px rgba(0,0,0,0.4); + } + .modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid #3a3a3a; + } + .modal-close-btn { + background: #f44336; + color: #fff; + border: none; + border-radius: 4px; + padding: 4px 8px; + cursor: pointer; + } + .modal-body { + flex: 1 1 auto; + overflow: auto; + padding: 12px 16px; + } + .modal-footer { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-top: 1px solid #3a3a3a; + } + .footer-left { color: #cccccc; font-style: italic; } -

Mindcraft

+
+
+

Mindcraft

+ mindserver offline +
+
+
-
-

Create Agent

-
-
Profile: Not uploaded
-
- - - +
+
+ +
+
+ + +
+
+ + + + + + diff --git a/src/mindcraft/public/settings_spec.json b/src/mindcraft/public/settings_spec.json index f003eff..431f89b 100644 --- a/src/mindcraft/public/settings_spec.json +++ b/src/mindcraft/public/settings_spec.json @@ -6,8 +6,8 @@ }, "minecraft_version": { "type": "string", - "description": "The version of Minecraft to use", - "default": "1.21.1" + "description": "The version of Minecraft to use. Set to 'auto' to automatically detect the version.", + "default": "auto" }, "host": { "type": "string", @@ -16,18 +16,20 @@ }, "port": { "type": "number", - "description": "The minecraft server port to connect to", + "description": "The minecraft server port to connect to. -1 for auto-detect.", "default": 55916 }, "auth": { "type": "string", "description": "The authentication method to use", - "default": "offline" + "default": "offline", + "options": ["offline", "microsoft"] }, "base_profile": { "type": "string", "description": "Allowed values: survival, assistant, creative, god_mode. Each has fine tuned settings for different game modes.", - "default": "survival" + "default": "survival", + "options": ["survival", "assistant", "creative", "god_mode"] }, "load_memory": { "type": "boolean", @@ -97,7 +99,13 @@ "show_command_syntax": { "type": "string", "description": "Whether to show \"full\" command syntax, \"shortened\" command syntax, or \"none\"", - "default": "full" + "default": "full", + "options": ["full", "shortened", "none"] + }, + "chat_ingame": { + "type": "boolean", + "description": "Whether to show bot's chat messages in game chat", + "default": true }, "chat_bot_messages": { "type": "boolean", diff --git a/src/models/_model_map.js b/src/models/_model_map.js index be43893..c119f07 100644 --- a/src/models/_model_map.js +++ b/src/models/_model_map.js @@ -55,7 +55,7 @@ export function selectAPI(profile) { else if (profile.model.includes('gemini')) profile.api = "google"; else if (profile.model.includes('grok')) - profile.api = 'grok'; + profile.api = 'xai'; else if (profile.model.includes('mistral')) profile.api = 'mistral'; else if (profile.model.includes('deepseek'))