From bdee71ac921b80a967f27daca643e26c4512ddc6 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 7 Mar 2025 14:19:55 -0600 Subject: [PATCH] camera always on, show entities, refactor browser viewer --- profiles/claude_thinker.json | 2 +- settings.js | 2 +- src/agent/agent.js | 9 ++-- .../{viewer.js => vision/browser_viewer.js} | 4 +- src/agent/vision/camera.js | 50 +++++++++---------- src/agent/vision/vision_interpreter.js | 15 ++---- 6 files changed, 38 insertions(+), 44 deletions(-) rename src/agent/{viewer.js => vision/browser_viewer.js} (69%) diff --git a/profiles/claude_thinker.json b/profiles/claude_thinker.json index aab9f10..fdddb59 100644 --- a/profiles/claude_thinker.json +++ b/profiles/claude_thinker.json @@ -6,7 +6,7 @@ "params": { "thinking": { "type": "enabled", - "budget_tokens": 16000 + "budget_tokens": 4000 } } }, diff --git a/settings.js b/settings.js index 1ae05a3..d3899b4 100644 --- a/settings.js +++ b/settings.js @@ -1,6 +1,6 @@ export default { - "minecraft_version": "1.20.4", // supports up to 1.21.1 + "minecraft_version": "1.21.1", // supports up to 1.21.1 "host": "127.0.0.1", // or "localhost", "your.ip.address.here" "port": process.env.MINECRAFT_PORT || 55916, "auth": "offline", // or "microsoft" diff --git a/src/agent/agent.js b/src/agent/agent.js index f186e52..fb123f6 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -11,7 +11,7 @@ import { MemoryBank } from './memory_bank.js'; import { SelfPrompter } from './self_prompter.js'; import convoManager from './conversation.js'; import { handleTranslation, handleEnglishTranslation } from '../utils/translator.js'; -import { addViewer } from './viewer.js'; +import { addBrowserViewer } from './vision/browser_viewer.js'; import settings from '../../settings.js'; import { serverProxy } from './agent_proxy.js'; import { Task } from './tasks.js'; @@ -36,8 +36,6 @@ export class Agent { this.history = new History(this); console.log('Initializing coder...'); this.coder = new Coder(this); - console.log('Initializing vision intepreter...'); - this.vision_interpreter = new VisionInterpreter(this, settings.allow_vision); console.log('Initializing npc controller...'); this.npc = new NPCContoller(this); console.log('Initializing memory bank...'); @@ -82,7 +80,7 @@ export class Agent { this.bot.once('spawn', async () => { try { clearTimeout(spawnTimeout); - addViewer(this.bot, count_id); + addBrowserViewer(this.bot, count_id); // wait for a bit so stats are not undefined await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -97,6 +95,9 @@ export class Agent { this.task.initBotTask(); } + console.log('Initializing vision intepreter...'); + this.vision_interpreter = new VisionInterpreter(this, settings.allow_vision); + } catch (error) { console.error('Error in spawn event:', error); process.exit(0); diff --git a/src/agent/viewer.js b/src/agent/vision/browser_viewer.js similarity index 69% rename from src/agent/viewer.js rename to src/agent/vision/browser_viewer.js index 6ce8a27..9ae7c7b 100644 --- a/src/agent/viewer.js +++ b/src/agent/vision/browser_viewer.js @@ -1,8 +1,8 @@ -import settings from '../../settings.js'; +import settings from '../../../settings.js'; import prismarineViewer from 'prismarine-viewer'; const mineflayerViewer = prismarineViewer.mineflayer; -export function addViewer(bot, count_id) { +export function addBrowserViewer(bot, count_id) { if (settings.show_bot_views) mineflayerViewer(bot, { port: 3000+count_id, firstPerson: true, }); } \ No newline at end of file diff --git a/src/agent/vision/camera.js b/src/agent/vision/camera.js index 7eafb42..c6d9487 100644 --- a/src/agent/vision/camera.js +++ b/src/agent/vision/camera.js @@ -14,37 +14,37 @@ global.Worker = worker_threads.Worker; export class Camera extends EventEmitter { constructor (bot, fp) { - super() - this.bot = bot - this.fp = fp - this.viewDistance = 4 - this.width = 800 - this.height = 512 - this.canvas = createCanvas(this.width, this.height) - this.renderer = new THREE.WebGLRenderer({ canvas: this.canvas }) - this.viewer = new Viewer(this.renderer) - this._init().then(() => { - this.emit('ready') - }) + super(); + this.bot = bot; + this.fp = fp; + this.viewDistance = 4; + this.width = 800; + this.height = 512; + this.canvas = createCanvas(this.width, this.height); + this.renderer = new THREE.WebGLRenderer({ canvas: this.canvas }); + this.viewer = new Viewer(this.renderer); + this._init().then(() => { + this.emit('ready'); + }) } async _init () { - const botPos = this.bot.entity.position - const center = new Vec3(botPos.x, botPos.y+this.bot.entity.height, botPos.z) - this.viewer.setVersion(this.bot.version) - // Load world - const worldView = new WorldView(this.bot.world, this.viewDistance, center) - this.viewer.listen(worldView) - - this.viewer.camera.position.set(center.x, center.y, center.z) - this.viewer.setFirstPersonCamera(this.bot.entity.position, this.bot.entity.yaw, this.bot.entity.pitch) - - await worldView.init(center) + const botPos = this.bot.entity.position; + const center = new Vec3(botPos.x, botPos.y+this.bot.entity.height, botPos.z); + this.viewer.setVersion(this.bot.version); + // Load world + const worldView = new WorldView(this.bot.world, this.viewDistance, center); + this.viewer.listen(worldView); + worldView.listenToBot(this.bot); + await worldView.init(center); + this.worldView = worldView; } async capture() { - // waits some time helps renderer to render the world view - await new Promise(resolve => setTimeout(resolve, 1000)); + const center = new Vec3(this.bot.entity.position.x, this.bot.entity.position.y+this.bot.entity.height, this.bot.entity.position.z); + this.viewer.camera.position.set(center.x, center.y, center.z); + this.viewer.setFirstPersonCamera(this.bot.entity.position, this.bot.entity.yaw, this.bot.entity.pitch); + this.viewer.update(); this.renderer.render(this.viewer.scene, this.viewer.camera); const imageStream = this.canvas.createJPEGStream({ diff --git a/src/agent/vision/vision_interpreter.js b/src/agent/vision/vision_interpreter.js index 34d9f3c..2c03276 100644 --- a/src/agent/vision/vision_interpreter.js +++ b/src/agent/vision/vision_interpreter.js @@ -2,13 +2,12 @@ import { Vec3 } from 'vec3'; import { Camera } from "./camera.js"; import fs from 'fs'; -const RENDER_TIME = 1000; - export class VisionInterpreter { constructor(agent, allow_vision) { this.agent = agent; this.allow_vision = allow_vision; this.fp = './bots/'+agent.name+'/screenshots/'; + this.camera = new Camera(agent.bot, this.fp); } async lookAtPlayer(player_name, direction) { @@ -25,16 +24,12 @@ export class VisionInterpreter { let filename; if (direction === 'with') { await bot.look(player.yaw, player.pitch); - const camera = new Camera(bot, this.fp); - await new Promise(resolve => setTimeout(resolve, RENDER_TIME)); result = `Looking in the same direction as ${player_name}\n`; - filename = await camera.capture(); + filename = await this.camera.capture(); } else { await bot.lookAt(new Vec3(player.position.x, player.position.y + player.height, player.position.z)); - const camera = new Camera(bot, this.fp); - await new Promise(resolve => setTimeout(resolve, RENDER_TIME)); result = `Looking at player ${player_name}\n`; - filename = await camera.capture(); + filename = await this.camera.capture(); } @@ -48,11 +43,9 @@ export class VisionInterpreter { let result = ""; const bot = this.agent.bot; await bot.lookAt(new Vec3(x, y + 2, z)); - const camera = new Camera(bot, this.fp); - await new Promise(resolve => setTimeout(resolve, RENDER_TIME)); result = `Looking at coordinate ${x, y, z}\n`; - let filename = await camera.capture(); + let filename = await this.camera.capture(); return result + `Image analysis: "${await this.analyzeImage(filename)}"`; }