From 2c1ff9e77d5ca34d31649fb32efa83fec7de6215 Mon Sep 17 00:00:00 2001
From: MaxRobinsonTheGreat <maxdrob1@gmail.com>
Date: Sat, 15 Mar 2025 18:10:31 -0500
Subject: [PATCH] get actual center block view/coords

---
 profiles/defaults/_default.json        |  2 +-
 src/agent/vision/vision_interpreter.js | 22 +++++++++++++++++++---
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/profiles/defaults/_default.json b/profiles/defaults/_default.json
index ab63863..fc2b60e 100644
--- a/profiles/defaults/_default.json
+++ b/profiles/defaults/_default.json
@@ -9,7 +9,7 @@
     
     "bot_responder": "You are a minecraft bot named $NAME that is currently in conversation with another AI bot. Both of you can take actions with the !command syntax, and actions take time to complete. You are currently busy with the following action: '$ACTION' but have received a new message. Decide whether to 'respond' immediately or 'ignore' it and wait for your current action to finish. Be conservative and only respond when necessary, like when you need to change/stop your action, or convey necessary information. Example 1: You:Building a house! !newAction('Build a house.').\nOther Bot: 'Come here!'\nYour decision: ignore\nExample 2: You:Collecting dirt !collectBlocks('dirt',10).\nOther Bot: 'No, collect some wood instead.'\nYour decision: respond\nExample 3: You:Coming to you now. !goToPlayer('billy',3).\nOther Bot: 'What biome are you in?'\nYour decision: respond\nActual Conversation: $TO_SUMMARIZE\nDecide by outputting ONLY 'respond' or 'ignore', nothing else. Your decision:",
 
-    "image_analysis": "You are a Minecraft bot named $NAME that has been given a screenshot of your current view. Analyze and summarize the view; describe terrain, blocks, entities, structures, and notable features. Focus on details relevant to the conversation. Note the view is glitchy; the sky color is always blue regardless of weather or time, dropped items are pink cubes, blocks below y=0 do not render. Estimate the x,y,z location of the block at your center view given your current position. Be extremely concise and correct, respond only with your analysis, not conversationally. $STATS",
+    "image_analysis": "You are a Minecraft bot named $NAME that has been given a screenshot of your current view. Analyze and summarize the view; describe terrain, blocks, entities, structures, and notable features. Focus on details relevant to the conversation. Note: the sky is always blue regardless of weather or time, dropped items are small pink cubes, and blocks below y=0 do not render. Be extremely concise and correct, respond only with your analysis, not conversationally. $STATS",
 
     "modes": {
         "self_preservation": true,
diff --git a/src/agent/vision/vision_interpreter.js b/src/agent/vision/vision_interpreter.js
index 2c03276..a43acd2 100644
--- a/src/agent/vision/vision_interpreter.js
+++ b/src/agent/vision/vision_interpreter.js
@@ -7,7 +7,9 @@ export class VisionInterpreter {
         this.agent = agent;
         this.allow_vision = allow_vision;
         this.fp = './bots/'+agent.name+'/screenshots/';
-        this.camera = new Camera(agent.bot, this.fp);
+        if (allow_vision) {
+            this.camera = new Camera(agent.bot, this.fp);
+        }
     }
 
     async lookAtPlayer(player_name, direction) {
@@ -43,19 +45,33 @@ export class VisionInterpreter {
         let result = "";
         const bot = this.agent.bot;
         await bot.lookAt(new Vec3(x, y + 2, z));
-        result = `Looking at coordinate ${x, y, z}\n`;
+        result = `Looking at coordinate ${x}, ${y}, ${z}\n`;
 
         let filename = await this.camera.capture();
 
         return result + `Image analysis: "${await this.analyzeImage(filename)}"`;
     }
 
+    getCenterBlockInfo() {
+        const bot = this.agent.bot;
+        const maxDistance = 128; // Maximum distance to check for blocks
+        const targetBlock = bot.blockAtCursor(maxDistance);
+        
+        if (targetBlock) {
+            return `Block at center view: ${targetBlock.name} at (${targetBlock.position.x}, ${targetBlock.position.y}, ${targetBlock.position.z})`;
+        } else {
+            return "No block in center view";
+        }
+    }
+
     async analyzeImage(filename) {
         try {
             const imageBuffer = fs.readFileSync(`${this.fp}/${filename}.jpg`);
             const messages = this.agent.history.getHistory();
 
-            return await this.agent.prompter.promptVision(messages, imageBuffer);
+            const blockInfo = this.getCenterBlockInfo();
+            const result = await this.agent.prompter.promptVision(messages, imageBuffer);
+            return result + `\n${blockInfo}`;
 
         } catch (error) {
             console.warn('Error reading image:', error);