fix: add camera file & move image describe to promptImageConvo

profiles/_default.json

@@ -80,6 +80,12 @@
             {"role": "system", "content": "Code Output:\nLooking in the same direction as d7297\nThere is a castle made of stone."},
             {"role": "assistant", "content": "Wow, that stone castle looks amazing!"}
         ],
+        [
+            {"role": "user", "content": "d7297: look 0 70 12 and describe what is there"},
+            {"role": "assistant", "content": "Sure! !lookAtPosition(0, 70, 12)"},
+            {"role": "system", "content": "Code Output:\nLooking in the same direction as d7297\nThere is a water fountain."},
+            {"role": "assistant", "content": "I see the water fountain! amazing!"}
+        ],
 
         [
             {"role": "user", "content": "greg: Collect 10 wood"},

src/agent/prompter.js

@@ -287,6 +287,29 @@ export class Prompter {
         );
     }
 
+    async promptImageConvo(messages, imageBuffer) {
+        await this.checkCooldown();
+        let prompt = this.profile.image_conversing;
+        let imageMessages = [...messages];
+        imageMessages.push({
+            role: "user", 
+            content: [
+                { type: "text", text: "Briefly describe the screen you are looking at now." },
+                {
+                    type: "image_url",
+                    image_url: {
+                        "url": `data:image/jpeg;base64,${imageBuffer.toString('base64')}`,
+                    }
+                }
+            ]
+        });
+
+        return await this.chat_model.sendRequest(
+            imageMessages,
+            prompt
+        );
+    }
+
     async promptCoding(messages) {
         if (this.awaiting_coding) {
             console.warn('Already awaiting coding response, returning no response.');

src/utils/camera.js

@@ -0,0 +1,88 @@
+import { Viewer } from 'prismarine-viewer/viewer/lib/viewer.js';
+import { WorldView } from 'prismarine-viewer/viewer/lib/worldview.js';
+import { getBufferFromStream } from 'prismarine-viewer/viewer/lib/simpleUtils.js';
+
+import THREE from 'three';
+import { createCanvas } from 'node-canvas-webgl/lib/index.js';
+import fs from 'fs/promises';
+import { Vec3 } from 'vec3';
+import { EventEmitter } from 'events';
+
+import worker_threads from 'worker_threads';
+global.Worker = worker_threads.Worker;
+
+
+export class Camera extends EventEmitter {
+    constructor (bot) {
+      super()
+      this.bot = bot
+      this.viewDistance = 4
+      this.width = 800
+      this.height = 512
+      this.canvas = createCanvas(this.width, this.height)
+      this.renderer = new THREE.WebGLRenderer({ canvas: this.canvas })
+      this.viewer = new Viewer(this.renderer)
+      this._init().then(() => {
+        this.emit('ready')
+      })
+    }
+  
+    async _init () {
+      const botPos = this.bot.entity.position
+      const center = new Vec3(botPos.x, botPos.y+this.bot.entity.height, botPos.z)
+      this.viewer.setVersion(this.bot.version)      
+      // Load world
+      const worldView = new WorldView(this.bot.world, this.viewDistance, center)
+      this.viewer.listen(worldView)
+  
+      this.viewer.camera.position.set(center.x, center.y, center.z)
+  
+      await worldView.init(center)
+    }
+  
+    async captureTargetPoint(x, y, z) {  
+        this.viewer.camera.lookAt(x, y, z);
+        const filename = await this._capture();
+        return filename;
+    }
+  
+    async captureDirection(yaw, pitch) {
+        this.viewer.camera.rotation.y = yaw;
+        this.viewer.camera.rotation.x = pitch;
+        const filename = await this._capture();
+        return filename;
+    }
+
+    async _capture() {
+        console.info('Waiting for camera to load');
+        await new Promise(resolve => setTimeout(resolve, 5000));
+        this.renderer.render(this.viewer.scene, this.viewer.camera);
+
+        const imageStream = this.canvas.createJPEGStream({
+            bufsize: 4096,
+            quality: 100,
+            progressive: false
+        });
+        
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const filename = `screenshot_${timestamp}`;
+
+        const buf = await getBufferFromStream(imageStream);
+        await this._ensureScreenshotDirectory();
+        await fs.writeFile(`bots/${this.bot.username}/screenshots/${filename}.jpg`, buf);
+        console.log('saved', filename);
+        return filename;
+    }
+
+    async _ensureScreenshotDirectory() {
+        let stats;
+        try {
+            stats = await fs.stat(`bots/${this.bot.username}/screenshots`);
+        } catch (e) {
+            if (!stats?.isDirectory()) {
+                await fs.mkdir(`bots/${this.bot.username}/screenshots`);
+            }
+        }
+    }
+}
+