Merge branch 'main' into always-active-vision

logger.js

@@ -0,0 +1,401 @@
+// --- START OF FILE logger.js ---
+
+import { writeFileSync, mkdirSync, existsSync, appendFileSync, readFileSync } from 'fs';
+import { join } from 'path';
+import settings from './settings.js'; // Import settings
+import path from 'path'; // Needed for path operations
+
+// --- Configuration ---
+const LOGS_DIR = './logs';
+const VISION_DATASET_DIR = join(LOGS_DIR, 'vision_dataset'); // HuggingFace dataset format
+const VISION_IMAGES_DIR = join(VISION_DATASET_DIR, 'images'); // Images subdirectory
+
+// --- Log File Paths ---
+const REASONING_LOG_FILE = join(LOGS_DIR, 'reasoning_logs.csv');
+const NORMAL_LOG_FILE = join(LOGS_DIR, 'normal_logs.csv');
+const VISION_METADATA_FILE = join(VISION_DATASET_DIR, 'metadata.jsonl'); // HF metadata format
+
+// --- Log Headers ---
+const TEXT_LOG_HEADER = 'input,output\n';
+
+// --- Log Counters ---
+let logCounts = {
+    normal: 0,
+    reasoning: 0,
+    vision: 0,
+    total: 0,
+    skipped_disabled: 0,
+    skipped_empty: 0,
+    vision_images_saved: 0,
+};
+
+// --- Helper Functions ---
+function ensureDirectoryExistence(dirPath) {
+    if (!existsSync(dirPath)) {
+        try {
+            mkdirSync(dirPath, { recursive: true });
+            console.log(`[Logger] Created directory: ${dirPath}`);
+        } catch (error) {
+            console.error(`[Logger] Error creating directory ${dirPath}:`, error);
+            return false;
+        }
+    }
+    return true;
+}
+
+function countLogEntries(logFile) {
+    if (!existsSync(logFile)) return 0;
+    try {
+        const data = readFileSync(logFile, 'utf8');
+        const lines = data.split('\n').filter(line => line.trim());
+        // Check if the first line looks like a header before subtracting
+        const hasHeader = lines.length > 0 && lines[0].includes(',');
+        return Math.max(0, hasHeader ? lines.length - 1 : lines.length);
+    } catch (err) {
+        console.error(`[Logger] Error reading log file ${logFile}:`, err);
+        return 0;
+    }
+}
+
+
+function ensureLogFile(logFile, header) {
+     if (!ensureDirectoryExistence(path.dirname(logFile))) return false; // Ensure parent dir exists
+
+     if (!existsSync(logFile)) {
+        try {
+            writeFileSync(logFile, header);
+            console.log(`[Logger] Created log file: ${logFile}`);
+        } catch (error) {
+            console.error(`[Logger] Error creating log file ${logFile}:`, error);
+            return false;
+        }
+    } else {
+         try {
+            const content = readFileSync(logFile, 'utf-8');
+            const headerLine = header.split('\n')[0];
+            // If file is empty or header doesn't match, overwrite/create header
+            if (!content.trim() || !content.startsWith(headerLine)) {
+                 // Attempt to prepend header if file has content but wrong/no header
+                 if(content.trim() && !content.startsWith(headerLine)) {
+                    console.warn(`[Logger] Log file ${logFile} seems to be missing or has an incorrect header. Prepending correct header.`);
+                    writeFileSync(logFile, header + content);
+                 } else {
+                    // File is empty or correctly headed, just ensure header is there
+                     writeFileSync(logFile, header);
+                 }
+                 console.log(`[Logger] Ensured header in log file: ${logFile}`);
+            }
+        } catch (error) {
+            console.error(`[Logger] Error checking/writing header for log file ${logFile}:`, error);
+            // Proceed cautiously, maybe log an error and continue?
+        }
+    }
+    return true;
+}
+
+
+function writeToLogFile(logFile, csvEntry) {
+    try {
+        appendFileSync(logFile, csvEntry);
+        // console.log(`[Logger] Logged data to ${logFile}`); // Keep console less noisy
+    } catch (error) {
+        console.error(`[Logger] Error writing to CSV log file ${logFile}:`, error);
+    }
+}
+
+// --- Auto-Detection for Log Type (Based on Response Content) ---
+function determineLogType(response) {
+    // Reasoning check: needs <think>...</think> but ignore the specific 'undefined' placeholder
+    const isReasoning = response.includes('<think>') && response.includes('</think>') && !response.includes('<think>\nundefined</think>');
+
+    if (isReasoning) {
+        return 'reasoning';
+    } else {
+        return 'normal';
+    }
+}
+
+function sanitizeForCsv(value) {
+    if (typeof value !== 'string') {
+        value = String(value);
+    }
+    // Escape double quotes by doubling them and enclose the whole string in double quotes
+    return `"${value.replace(/"/g, '""')}"`;
+}
+
+// Helper function to clean reasoning markers from input
+function cleanReasoningMarkers(input) {
+    if (typeof input !== 'string') {
+        return input;
+    }
+    
+    // Remove /think and /no_think markers
+    return input.replace(/\/think/g, '').replace(/\/no_think/g, '').trim();
+}
+
+// --- Main Logging Function (for text-based input/output) ---
+export function log(input, response) {
+    const trimmedInputStr = input ? (typeof input === 'string' ? input.trim() : JSON.stringify(input)) : "";
+    const trimmedResponse = response ? String(response).trim() : ""; // Ensure response is a string
+
+    // Clean reasoning markers from input before logging
+    const cleanedInput = cleanReasoningMarkers(trimmedInputStr);
+
+    // Basic filtering
+    if (!cleanedInput && !trimmedResponse) {
+        logCounts.skipped_empty++;
+        return;
+    }
+    if (cleanedInput === trimmedResponse) {
+         logCounts.skipped_empty++;
+        return;
+    }
+     // Avoid logging common error messages that aren't useful training data
+    const errorMessages = [
+        "My brain disconnected, try again.",
+        "My brain just kinda stopped working. Try again.",
+        "I thought too hard, sorry, try again.",
+        "*no response*",
+        "No response received.",
+        "No response data.",
+        "Failed to send", // Broader match
+        "Error:", // Broader match
+        "Vision is only supported",
+        "Context length exceeded",
+        "Image input modality is not enabled",
+        "An unexpected error occurred",
+        // Add more generic errors/placeholders as needed
+    ];
+    // Also check for responses that are just the input repeated (sometimes happens with errors)
+    if (errorMessages.some(err => trimmedResponse.includes(err)) || trimmedResponse === cleanedInput) {
+        logCounts.skipped_empty++;
+        // console.warn(`[Logger] Skipping log due to error/placeholder/repeat: "${trimmedResponse.substring(0, 70)}..."`);
+        return;
+    }
+
+
+    const logType = determineLogType(trimmedResponse);
+    let logFile;
+    let header;
+    let settingFlag;
+
+    switch (logType) {
+        case 'reasoning':
+            logFile = REASONING_LOG_FILE;
+            header = TEXT_LOG_HEADER;
+            settingFlag = settings.log_reasoning_data;
+            break;
+        case 'normal':
+        default:
+            logFile = NORMAL_LOG_FILE;
+            header = TEXT_LOG_HEADER;
+            settingFlag = settings.log_normal_data;
+            break;
+    }
+
+    // Check if logging for this type is enabled
+    if (!settingFlag) {
+        logCounts.skipped_disabled++;
+        return;
+    }
+
+    // Ensure directory and file exist
+    if (!ensureLogFile(logFile, header)) return; // ensureLogFile now checks parent dir too
+
+    // Prepare the CSV entry using the sanitizer with cleaned input
+    const safeInput = sanitizeForCsv(cleanedInput);
+    const safeResponse = sanitizeForCsv(trimmedResponse);
+    const csvEntry = `${safeInput},${safeResponse}\n`;
+
+    // Write to the determined log file
+    writeToLogFile(logFile, csvEntry);
+
+    // Update counts
+    logCounts[logType]++;
+    logCounts.total++; // Total here refers to text logs primarily
+
+    // Display summary periodically (based on total text logs)
+    if (logCounts.normal + logCounts.reasoning > 0 && (logCounts.normal + logCounts.reasoning) % 20 === 0) {
+       printSummary();
+    }
+}
+
+// --- Enhanced Vision Logging Function for HuggingFace Dataset Format ---
+export function logVision(conversationHistory, imageBuffer, response, visionMessage = null) {
+    if (!settings.log_vision_data) {
+        logCounts.skipped_disabled++;
+        return;
+    }
+
+    const trimmedResponse = response ? String(response).trim() : "";
+    
+    if (!conversationHistory || conversationHistory.length === 0 || !trimmedResponse || !imageBuffer) {
+        logCounts.skipped_empty++;
+        return;
+    }
+
+    // Filter out error messages
+    const errorMessages = [
+        "My brain disconnected, try again.",
+        "My brain just kinda stopped working. Try again.",
+        "I thought too hard, sorry, try again.",
+        "*no response*",
+        "No response received.",
+        "No response data.",
+        "Failed to send",
+        "Error:",
+        "Vision is only supported",
+        "Context length exceeded",
+        "Image input modality is not enabled",
+        "An unexpected error occurred",
+    ];
+    
+    if (errorMessages.some(err => trimmedResponse.includes(err))) {
+        logCounts.skipped_empty++;
+        return;
+    }
+
+    // Ensure directories exist
+    if (!ensureDirectoryExistence(VISION_DATASET_DIR)) return;
+    if (!ensureDirectoryExistence(VISION_IMAGES_DIR)) return;
+
+    try {
+        // Generate unique filename for the image
+        const timestamp = Date.now();
+        const randomSuffix = Math.random().toString(36).substring(2, 8);
+        const imageFilename = `vision_${timestamp}_${randomSuffix}.jpg`;
+        const imagePath = join(VISION_IMAGES_DIR, imageFilename);
+        const relativeImagePath = `images/${imageFilename}`; // Relative path for metadata
+
+        // Save the image
+        writeFileSync(imagePath, imageBuffer);
+        logCounts.vision_images_saved++;
+
+        // Extract the actual message sent with the image
+        // This is typically the vision prompt/instruction
+        let inputMessage = visionMessage;
+        if (!inputMessage && conversationHistory.length > 0) {
+            // Try to get the last user message or system message
+            const lastMessage = conversationHistory[conversationHistory.length - 1];
+            if (typeof lastMessage.content === 'string') {
+                inputMessage = lastMessage.content;
+            } else if (Array.isArray(lastMessage.content)) {
+                // Find text content in the message
+                const textContent = lastMessage.content.find(c => c.type === 'text');
+                inputMessage = textContent ? textContent.text : '';
+            }
+        }
+
+        // Fallback to conversation history if no specific message
+        if (!inputMessage) {
+            inputMessage = formatConversationInput(conversationHistory);
+        }
+
+        // Create metadata entry in JSONL format for HuggingFace
+        const metadataEntry = {
+            file_name: relativeImagePath,
+            text: inputMessage,
+            response: trimmedResponse,
+            timestamp: timestamp
+        };
+
+        // Append to metadata JSONL file
+        const jsonlLine = JSON.stringify(metadataEntry) + '\n';
+        appendFileSync(VISION_METADATA_FILE, jsonlLine);
+        
+        logCounts.vision++;
+        logCounts.total++;
+
+        // Display summary periodically
+        if (logCounts.vision > 0 && logCounts.vision % 10 === 0) {
+            printSummary();
+        }
+
+    } catch (error) {
+        console.error(`[Logger] Error logging vision data:`, error);
+    }
+}
+
+// Helper function to format conversation history as fallback
+function formatConversationInput(conversationHistory) {
+    if (!conversationHistory || conversationHistory.length === 0) return '';
+    
+    const formattedHistory = [];
+    
+    for (const turn of conversationHistory) {
+        const formattedTurn = {
+            role: turn.role || 'user',
+            content: []
+        };
+
+        // Handle different content formats
+        if (typeof turn.content === 'string') {
+            formattedTurn.content.push({
+                type: 'text',
+                text: turn.content
+            });
+        } else if (Array.isArray(turn.content)) {
+            // Already in the correct format
+            formattedTurn.content = turn.content;
+        } else if (turn.content && typeof turn.content === 'object') {
+            // Convert object to array format
+            if (turn.content.text) {
+                formattedTurn.content.push({
+                    type: 'text',
+                    text: turn.content.text
+                });
+            }
+            if (turn.content.image) {
+                formattedTurn.content.push({
+                    type: 'image',
+                    image: turn.content.image
+                });
+            }
+        }
+
+        formattedHistory.push(formattedTurn);
+    }
+    
+    return JSON.stringify(formattedHistory);
+}
+
+function printSummary() {
+    const totalStored = logCounts.normal + logCounts.reasoning + logCounts.vision;
+    console.log('\n' + '='.repeat(60));
+    console.log('LOGGER SUMMARY');
+    console.log('-'.repeat(60));
+    console.log(`Normal logs stored:    ${logCounts.normal}`);
+    console.log(`Reasoning logs stored: ${logCounts.reasoning}`);
+    console.log(`Vision logs stored:    ${logCounts.vision} (Images saved: ${logCounts.vision_images_saved})`);
+    console.log(`Skipped (disabled):    ${logCounts.skipped_disabled}`);
+    console.log(`Skipped (empty/err):   ${logCounts.skipped_empty}`);
+    console.log('-'.repeat(60));
+    console.log(`Total logs stored:     ${totalStored}`);
+    console.log('='.repeat(60) + '\n');
+}
+
+// Initialize counts at startup
+function initializeCounts() {
+    logCounts.normal = countLogEntries(NORMAL_LOG_FILE);
+    logCounts.reasoning = countLogEntries(REASONING_LOG_FILE);
+    logCounts.vision = countVisionEntries(VISION_METADATA_FILE);
+    // Total count will be accumulated during runtime
+    console.log(`[Logger] Initialized log counts: Normal=${logCounts.normal}, Reasoning=${logCounts.reasoning}, Vision=${logCounts.vision}`);
+}
+
+function countVisionEntries(metadataFile) {
+    if (!existsSync(metadataFile)) return 0;
+    try {
+        const data = readFileSync(metadataFile, 'utf8');
+        const lines = data.split('\n').filter(line => line.trim());
+        return lines.length;
+    } catch (err) {
+        console.error(`[Logger] Error reading vision metadata file ${metadataFile}:`, err);
+        return 0;
+    }
+}
+
+// Initialize counts at startup
+initializeCounts();
+
+// --- END OF FILE logger.js ---

settings.js

@@ -46,7 +46,11 @@ const settings = {
     "verbose_commands": true, // show full command syntax
     "narrate_behavior": true, // chat simple automatic actions ('Picking up item!')
     "chat_bot_messages": true, // publicly chat messages to other bots
-    "log_all_prompts": false, // log ALL prompts to file
+    
+    "log_normal_data": false,
+    "log_reasoning_data": false,
+    "log_vision_data": false,
+    
 }
 
 // these environment variables override certain settings
@@ -71,8 +75,5 @@ if (process.env.MAX_MESSAGES) {
 if (process.env.NUM_EXAMPLES) {
     settings.num_examples = process.env.NUM_EXAMPLES;
 }
-if (process.env.LOG_ALL) {
-    settings.log_all_prompts = process.env.LOG_ALL;
-}
 
 export default settings;

src/models/claude.js

@@ -1,18 +1,16 @@
 import Anthropic from '@anthropic-ai/sdk';
 import { strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Claude {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params || {};
-
         let config = {};
         if (url)
             config.baseURL = url;
-        
         config.apiKey = getKey('ANTHROPIC_API_KEY');
-
         this.anthropic = new Anthropic(config);
         this.supportsRawImageInput = true;
     }
@@ -71,8 +69,7 @@ export class Claude {
 
             if (!this.params.max_tokens) {
                 if (this.params.thinking?.budget_tokens) {
-                    this.params.max_tokens = this.params.thinking.budget_tokens + 1000;
-                    // max_tokens must be greater than thinking.budget_tokens
+                    this.params.max_tokens = this.params.thinking.budget_tokens + 1000; // max_tokens must be greater
                 } else {
                     this.params.max_tokens = 4096;
                 }
@@ -83,9 +80,7 @@ export class Claude {
                 messages: messages, // messages array is now potentially modified with image data
                 ...(this.params || {})
             });
-
             console.log('Received.')
-            // get first content of type text
             const textContent = resp.content.find(content => content.type === 'text');
             if (textContent) {
                 res = textContent.text;
@@ -93,8 +88,7 @@ export class Claude {
                 console.warn('No text content found in the response.');
                 res = 'No response from Claude.';
             }
-        }
-        catch (err) {
+        } catch (err) {
             if (err.message.includes("does not support image input")) {
                 res = "Vision is only supported by certain models.";
             } else {
@@ -102,30 +96,34 @@ export class Claude {
             }
             console.log(err);
         }
+        const logMessagesForClaude = [{ role: "system", content: systemMessage }].concat(turns);
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(logMessagesForClaude), res);
         return res;
     }
 
     async sendVisionRequest(turns, systemMessage, imageBuffer) {
-        const imageMessages = [...turns];
-        imageMessages.push({
-            role: "user",
-            content: [
-                {
-                    type: "text",
-                    text: systemMessage
-                },
-                {
-                    type: "image",
-                    source: {
-                        type: "base64",
-                        media_type: "image/jpeg",
-                        data: imageBuffer.toString('base64')
-                    }
+        const visionUserMessageContent = [
+            { type: "text", text: systemMessage },
+            {
+                type: "image",
+                source: {
+                    type: "base64",
+                    media_type: "image/jpeg",
+                    data: imageBuffer.toString('base64')
                 }
-            ]
-        });
+            }
+        ];
+        const turnsForAPIRequest = [...turns, { role: "user", content: visionUserMessageContent }];
 
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(turnsForAPIRequest, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {

src/models/deepseek.js

@@ -1,17 +1,15 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class DeepSeek {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
-
         let config = {};
-
         config.baseURL = url || 'https://api.deepseek.com';
         config.apiKey = getKey('DEEPSEEK_API_KEY');
-
         this.openai = new OpenAIApi(config);
         this.supportsRawImageInput = true; // Assuming DeepSeek models used can support this OpenAI-like format
     }
@@ -78,18 +76,17 @@ export class DeepSeek {
             stop: stop_seq,
             ...(this.params || {})
         };
-
         let res = null;
         try {
-            console.log('Awaiting deepseek api response...');
-            // console.log('Formatted Messages for API:', JSON.stringify(messages, null, 2));
+          
+            console.log('Awaiting deepseek api response...')
+          
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded');
             console.log('Received.');
             res = completion.choices[0].message.content;
-        }
-        catch (err) {
+        } catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
@@ -98,6 +95,10 @@ export class DeepSeek {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 
@@ -105,6 +106,3 @@ export class DeepSeek {
         throw new Error('Embeddings are not supported by Deepseek.');
     }
 }
-
-
-

src/models/gemini.js

@@ -1,6 +1,7 @@
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import { toSinglePrompt, strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Gemini {
     constructor(model_name, url, params) {
@@ -8,28 +9,12 @@ export class Gemini {
         this.params = params;
         this.url = url;
         this.safetySettings = [
-            {
-                "category": "HARM_CATEGORY_DANGEROUS",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_HARASSMENT",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_HATE_SPEECH",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
-                "threshold": "BLOCK_NONE",
-            },
+            { "category": "HARM_CATEGORY_DANGEROUS", "threshold": "BLOCK_NONE" },
+            { "category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE" },
+            { "category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE" },
+            { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE" },
+            { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE" },
         ];
-
         this.genAI = new GoogleGenerativeAI(getKey('GEMINI_API_KEY'));
         this.supportsRawImageInput = true;
     }
@@ -41,20 +26,12 @@ export class Gemini {
             // systemInstruction does not work bc google is trash
         };
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
-                modelConfig,
-                { baseUrl: this.url },
-                { safetySettings: this.safetySettings }
-            );
+            model = this.genAI.getGenerativeModel(modelConfig, { baseUrl: this.url }, { safetySettings: this.safetySettings });
         } else {
-            model = this.genAI.getGenerativeModel(
-                modelConfig,
-                { safetySettings: this.safetySettings }
-            );
+            model = this.genAI.getGenerativeModel(modelConfig, { safetySettings: this.safetySettings });
         }
-
         console.log('Awaiting Google API response...');
-
+        const originalTurnsForLog = [{role: 'system', content: systemMessage}, ...turns];
         turns.unshift({ role: 'system', content: systemMessage });
         turns = strictFormat(turns);
         let contents = [];
@@ -85,22 +62,12 @@ export class Gemini {
 
         const result = await model.generateContent({
             contents,
-            generationConfig: {
-                ...(this.params || {})
-            }
+            generationConfig: { ...(this.params || {}) }
         });
         const response = await result.response;
         let text;
-
-        // Handle "thinking" models since they smart 
         if (this.model_name && this.model_name.includes("thinking")) {
-            if (
-                response.candidates &&
-                response.candidates.length > 0 &&
-                response.candidates[0].content &&
-                response.candidates[0].content.parts &&
-                response.candidates[0].content.parts.length > 1
-            ) {
+            if (response.candidates?.length > 0 && response.candidates[0].content?.parts?.length > 1) {
                 text = response.candidates[0].content.parts[1].text;
             } else {
                 console.warn("Unexpected response structure for thinking model:", response);
@@ -109,34 +76,22 @@ export class Gemini {
         } else {
             text = response.text();
         }
-
         console.log('Received.');
-
+        if (typeof text === 'string') {
+            text = text.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(originalTurnsForLog), text);
         return text;
     }
 
     async sendVisionRequest(turns, systemMessage, imageBuffer) {
         let model;
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
-                { model: this.model_name || "gemini-1.5-flash" },
-                { baseUrl: this.url },
-                { safetySettings: this.safetySettings }
-            );
+            model = this.genAI.getGenerativeModel({ model: this.model_name || "gemini-1.5-flash" }, { baseUrl: this.url }, { safetySettings: this.safetySettings });
         } else {
-            model = this.genAI.getGenerativeModel(
-                { model: this.model_name || "gemini-1.5-flash" },
-                { safetySettings: this.safetySettings }
-            );
+            model = this.genAI.getGenerativeModel({ model: this.model_name || "gemini-1.5-flash" }, { safetySettings: this.safetySettings });
         }
-
-        const imagePart = {
-            inlineData: {
-                data: imageBuffer.toString('base64'),
-                mimeType: 'image/jpeg'
-            }
-        };
-
+        const imagePart = { inlineData: { data: imageBuffer.toString('base64'), mimeType: 'image/jpeg' } };
         const stop_seq = '***';
         const prompt = toSinglePrompt(turns, systemMessage, stop_seq, 'model');
         let res = null;
@@ -146,6 +101,9 @@ export class Gemini {
             const response = await result.response;
             const text = response.text();
             console.log('Received.');
+            if (imageBuffer && text) {
+                logVision(turns, imageBuffer, text, prompt);
+            }
             if (!text.includes(stop_seq)) return text;
             const idx = text.indexOf(stop_seq);
             res = text.slice(0, idx);
@@ -156,6 +114,11 @@ export class Gemini {
             } else {
                 res = "An unexpected error occurred, please try again.";
             }
+            const loggedTurnsForError = [{role: 'system', content: systemMessage}, ...turns];
+            if (typeof res === 'string') {
+                res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+            }
+            log(JSON.stringify(loggedTurnsForError), res);
         }
         return res;
     }
@@ -163,16 +126,10 @@ export class Gemini {
     async embed(text) {
         let model;
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
-                { model: "text-embedding-004" },
-                { baseUrl: this.url }
-            );
+            model = this.genAI.getGenerativeModel({ model: "text-embedding-004" }, { baseUrl: this.url });
         } else {
-            model = this.genAI.getGenerativeModel(
-                { model: "text-embedding-004" }
-            );
+            model = this.genAI.getGenerativeModel({ model: "text-embedding-004" });
         }
-
         const result = await model.embedContent(text);
         return result.embedding.values;
     }

src/models/glhf.js

@@ -1,5 +1,6 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class GLHF {
     constructor(model_name, url) {
@@ -48,11 +49,13 @@ export class GLHF {
                     continue;
                 }
                 // If there's a closing </think> tag but no opening <think>, prepend one.
+
                 if (res.includes("</think>") && !res.includes("<think>")) {
                     res = "<think>" + res;
                 }
                 finalRes = res.replace(/<\|separator\|>/g, '*no response*');
                 break; // Valid response obtained.
+              
             } catch (err) {
                 if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) {
                     console.log('Context length exceeded, trying again with shorter context.');
@@ -68,6 +71,11 @@ export class GLHF {
         if (finalRes === null) {
             finalRes = "I thought too hard, sorry, try again";
         }
+
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/gpt.js

@@ -1,21 +1,18 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class GPT {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
-
         let config = {};
         if (url)
             config.baseURL = url;
-
         if (hasKey('OPENAI_ORG_ID'))
             config.organization = getKey('OPENAI_ORG_ID');
-
         config.apiKey = getKey('OPENAI_API_KEY');
-
         this.openai = new OpenAIApi(config);
         this.supportsRawImageInput = true;
     }
@@ -65,19 +62,17 @@ export class GPT {
         if (this.model_name.includes('o1')) {
             delete pack.stop;
         }
-
         let res = null;
-
         try {
+
             console.log('Awaiting openai api response from model', this.model_name);
-            // console.log('Formatted Messages for API:', JSON.stringify(messages, null, 2));
+
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded');
             console.log('Received.');
             res = completion.choices[0].message.content;
-        }
-        catch (err) {
+        } catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
@@ -89,25 +84,32 @@ export class GPT {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
-        imageMessages.push({
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
+        const imageFormattedTurns = [...original_turns];
+        imageFormattedTurns.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
                 }
             ]
         });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {
@@ -120,8 +122,4 @@ export class GPT {
         });
         return embedding.data[0].embedding;
     }
-
 }
-
-
-

src/models/grok.js

@@ -1,5 +1,6 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // xAI doesn't supply a SDK for their models, but fully supports OpenAI and Anthropic SDKs
 export class Grok {
@@ -7,15 +8,12 @@ export class Grok {
         this.model_name = model_name;
         this.url = url;
         this.params = params;
-
         let config = {};
         if (url)
             config.baseURL = url;
         else
             config.baseURL = "https://api.x.ai/v1"
-
         config.apiKey = getKey('XAI_API_KEY');
-
         this.openai = new OpenAIApi(config);
         // Direct image data in sendRequest is not supported by this wrapper for standard chat.
         // Grok may have specific vision capabilities, but this method assumes text-only.
@@ -27,25 +25,21 @@ export class Grok {
             console.warn(`[Grok] Warning: imageData provided to sendRequest, but this method in grok.js does not support direct image data embedding for model ${this.model_name}. The image will be ignored.`);
         }
         let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
-
         const pack = {
             model: this.model_name || "grok-beta",
             messages,
             stop: [stop_seq],
             ...(this.params || {})
         };
-
         let res = null;
         try {
             console.log('Awaiting xai api response...')
-            ///console.log('Messages:', messages);
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded'); 
             console.log('Received.')
             res = completion.choices[0].message.content;
-        }
-        catch (err) {
+        } catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, imageData, stop_seq);
@@ -58,31 +52,36 @@ export class Grok {
             }
         }
         // sometimes outputs special token <|separator|>, just replace it
-        return res.replace(/<\|separator\|>/g, '*no response*');
+        let finalResponseText = res ? res.replace(/<\|separator\|>/g, '*no response*') : (res === null ? "*no response*" : res);
+        if (typeof finalResponseText === 'string') {
+            finalResponseText = finalResponseText.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalResponseText);
+        return finalResponseText;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
-        imageMessages.push({
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
+        const imageFormattedTurns = [...original_turns];
+        imageFormattedTurns.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
                 }
             ]
         });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
     
     async embed(text) {
         throw new Error('Embeddings are not supported by Grok.');
     }
 }
-
-
-

src/models/groq.js

@@ -1,14 +1,13 @@
 import Groq from 'groq-sdk'
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // THIS API IS NOT TO BE CONFUSED WITH GROK!
 // Go to grok.js for that. :)
 
 // Umbrella class for everything under the sun... That GroqCloud provides, that is.
 export class GroqCloudAPI {
-
     constructor(model_name, url, params) {
-
         this.model_name = model_name;
         this.url = url;
         this.params = params || {};
@@ -18,7 +17,6 @@ export class GroqCloudAPI {
             delete this.params.tools;
         // This is just a bit of future-proofing in case we drag Mindcraft in that direction.
 
-        // I'm going to do a sneaky ReplicateAPI theft for a lot of this, aren't I?
         if (this.url)
             console.warn("Groq Cloud has no implementation for custom URLs. Ignoring provided URL.");
 
@@ -35,9 +33,7 @@ export class GroqCloudAPI {
         }
         // Construct messages array
         let messages = [{"role": "system", "content": systemMessage}].concat(turns);
-
         let res = null;
-
         try {
             console.log("Awaiting Groq response...");
 
@@ -47,7 +43,6 @@ export class GroqCloudAPI {
                 this.params.max_completion_tokens = this.params.max_tokens;
                 delete this.params.max_tokens;
             }
-
             if (!this.params.max_completion_tokens) {
                 this.params.max_completion_tokens = 4000;
             }
@@ -60,11 +55,15 @@ export class GroqCloudAPI {
                 ...(this.params || {})
             });
 
-            res = completion.choices[0].message;
-
-            res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
-        }
-        catch(err) {
+            let responseText = completion.choices[0].message.content;
+            if (typeof responseText === 'string') {
+                responseText = responseText.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+            }
+            log(JSON.stringify(messages), responseText);
+            // Original cleaning of <think> tags for the *returned* response (not affecting log)
+            responseText = responseText.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+            return responseText;
+        } catch(err) {
             if (err.message.includes("content must be a string")) {
                 res = "Vision is only supported by certain models.";
             } else {
@@ -72,27 +71,33 @@ export class GroqCloudAPI {
                 res = "My brain disconnected, try again.";
             }
             console.log(err);
+            if (typeof res === 'string') {
+                res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+            }
+            log(JSON.stringify(messages), res);
+            return res;
         }
-        return res;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
-        const imageMessages = messages.filter(message => message.role !== 'system');
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
+        const imageMessages = [...original_turns];
         imageMessages.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
                 }
             ]
         });
-        
-        // sendVisionRequest formats its own message array; sendRequest here should not process new imageData.
-        return this.sendRequest(imageMessages, systemMessage, null, stop_seq);
+
+        const res = await this.sendRequest(imageMessages, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(_) {

src/models/huggingface.js

@@ -1,18 +1,16 @@
 import { toSinglePrompt } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
 import { HfInference } from "@huggingface/inference";
+import { log, logVision } from '../../logger.js';
 
 export class HuggingFace {
   constructor(model_name, url, params) {
-    // Remove 'huggingface/' prefix if present
     this.model_name = model_name.replace('huggingface/', '');
     this.url = url;
     this.params = params;
-
     if (this.url) {
       console.warn("Hugging Face doesn't support custom urls!");
     }
-
     this.huggingface = new HfInference(getKey('HUGGINGFACE_API_KEY'));
     // Direct image data in sendRequest is not supported by this wrapper.
     // HuggingFace Inference API has other methods for vision tasks.
@@ -24,14 +22,11 @@ export class HuggingFace {
       console.warn(`[HuggingFace] Warning: imageData provided to sendRequest, but this method in huggingface.js does not support direct image data embedding for model ${this.model_name}. The image will be ignored.`);
     }
     const stop_seq = '***';
-    // Build a single prompt from the conversation turns
     const prompt = toSinglePrompt(turns, null, stop_seq);
-    // Fallback model if none was provided
     const model_name = this.model_name || 'meta-llama/Meta-Llama-3-8B';
-    // Combine system message with the prompt
-    const input = systemMessage + "\n" + prompt;
-
-    // We'll try up to 5 times in case of partial <think> blocks for DeepSeek-R1 models.
+    const logInputMessages = [{role: 'system', content: systemMessage}, ...turns];
+    const input = systemMessage + "
+" + prompt;
     const maxAttempts = 5;
     let attempt = 0;
     let finalRes = null;
@@ -41,7 +36,6 @@ export class HuggingFace {
       console.log(`Awaiting Hugging Face API response... (model: ${model_name}, attempt: ${attempt})`);
       let res = '';
       try {
-        // Consume the streaming response chunk by chunk
         for await (const chunk of this.huggingface.chatCompletionStream({
           model: model_name,
           messages: [{ role: "user", content: input }],
@@ -52,36 +46,32 @@ export class HuggingFace {
       } catch (err) {
         console.log(err);
         res = 'My brain disconnected, try again.';
-        // Break out immediately; we only retry when handling partial <think> tags.
         break;
       }
 
-      // If the model is DeepSeek-R1, check for mismatched <think> blocks.
-        const hasOpenTag = res.includes("<think>");
-        const hasCloseTag = res.includes("</think>");
-
-        // If there's a partial mismatch, warn and retry the entire request.
-        if ((hasOpenTag && !hasCloseTag)) {
-          console.warn("Partial <think> block detected. Re-generating...");
-          continue;
-        }
-
-        // If both tags are present, remove the <think> block entirely.
-        if (hasOpenTag && hasCloseTag) {
-          res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
-        }
+      const hasOpenTag = res.includes("<think>");
+      const hasCloseTag = res.includes("</think>");
 
+      if ((hasOpenTag && !hasCloseTag)) {
+        console.warn("Partial <think> block detected. Re-generating...");
+        if (attempt < maxAttempts) continue;
+      }
+      if (hasOpenTag && hasCloseTag) {
+        res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+      }
       finalRes = res;
-      break; // Exit loop if we got a valid response.
+      break;
     }
 
-    // If no valid response was obtained after max attempts, assign a fallback.
     if (finalRes == null) {
-      console.warn("Could not get a valid <think> block or normal response after max attempts.");
+      console.warn("Could not get a valid response after max attempts.");
       finalRes = 'I thought too hard, sorry, try again.';
     }
     console.log('Received.');
-    console.log(finalRes);
+    if (typeof finalRes === 'string') {
+        finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+    }
+    log(JSON.stringify(logInputMessages), finalRes);
     return finalRes;
   }
 

src/models/hyperbolic.js

@@ -1,4 +1,5 @@
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Hyperbolic {
     constructor(modelName, apiUrl) {
@@ -9,6 +10,7 @@ export class Hyperbolic {
         if (!this.apiKey) {
             throw new Error('HYPERBOLIC_API_KEY not found. Check your keys.js file.');
         }
+
         // Direct image data in sendRequest is not supported by this wrapper.
         this.supportsRawImageInput = false;
     }
@@ -38,14 +40,13 @@ export class Hyperbolic {
         const maxAttempts = 5;
         let attempt = 0;
         let finalRes = null;
+        let rawCompletionContent = null;
 
         while (attempt < maxAttempts) {
             attempt++;
             console.log(`Awaiting Hyperbolic API response... (attempt: ${attempt})`);
-            // console.log('Messages:', messages); // Avoid logging full messages in production if sensitive
-
-            let completionContent = null;
 
+          
             try {
                 const response = await fetch(this.apiUrl, {
                     method: 'POST',
@@ -65,55 +66,57 @@ export class Hyperbolic {
                     throw new Error(`HTTP error! status: ${response.status}, message: ${errorBody}`);
                 }
 
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
                 const data = await response.json();
                 if (data?.choices?.[0]?.finish_reason === 'length') {
                     throw new Error('Context length exceeded');
                 }
 
-                completionContent = data?.choices?.[0]?.message?.content || '';
+                rawCompletionContent = data?.choices?.[0]?.message?.content || '';
                 console.log('Received response from Hyperbolic.');
             } catch (err) {
-                if (
-                    (err.message.includes('Context length exceeded') || err.code === 'context_length_exceeded') && // Adjusted to check includes for message
-                    turns.length > 1
-                ) {
+                if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) {
                     console.log('Context length exceeded, trying again with a shorter context...');
-                    return await this.sendRequest(turns.slice(1), systemMessage, imageData, stopSeq); // Pass imageData
+                    return await this.sendRequest(turns.slice(1), systemMessage, imageData, stopSeq);
                 } else {
                     console.error(err);
-                    completionContent = 'My brain disconnected, try again.';
-                    // No break here, let it be set and then break after the think block logic
+                    rawCompletionContent = 'My brain disconnected, try again.';
+                    finalRes = rawCompletionContent;
+                    break;
                 }
             }
 
-            const hasOpenTag = completionContent.includes("<think>");
-            const hasCloseTag = completionContent.includes("</think>");
+            let processedContent = rawCompletionContent;
+            const hasOpenTag = processedContent.includes("<think>");
+            const hasCloseTag = processedContent.includes("</think>");
 
             if ((hasOpenTag && !hasCloseTag)) {
                 console.warn("Partial <think> block detected. Re-generating...");
-                if (attempt >= maxAttempts) { // If this was the last attempt
-                    finalRes = "I thought too hard and got stuck in a loop, sorry, try again.";
-                    break;
-                }
-                continue;
+                if (attempt < maxAttempts) continue;
             }
-
             if (hasCloseTag && !hasOpenTag) {
-                completionContent = '<think>' + completionContent;
+                processedContent = '<think>' + processedContent;
             }
-
             if (hasOpenTag && hasCloseTag) {
-                completionContent = completionContent.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+                processedContent = processedContent.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+            }
+            finalRes = processedContent.replace(/<\|separator\|>/g, '*no response*');
+            if (!(hasOpenTag && !hasCloseTag && attempt < maxAttempts)) {
+                break;
             }
-
-            finalRes = completionContent.replace(/<\|separator\|>/g, '*no response*');
-            break;
         }
 
-        if (finalRes == null) { // This condition might be hit if all attempts fail and continue
-            console.warn("Could not get a valid <think> block or normal response after max attempts.");
-            finalRes = 'I thought too hard, sorry, try again.';
+        if (finalRes == null) {
+            finalRes = rawCompletionContent || 'I thought too hard, sorry, try again.';
+            finalRes = finalRes.replace(/<\|separator\|>/g, '*no response*');
         }
+
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/local.js

@@ -1,4 +1,5 @@
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Local {
     constructor(model_name, url, params) {
@@ -38,7 +39,6 @@ export class Local {
             }
         }
         
-        // We'll attempt up to 5 times for models with deepseek-r1-esk reasoning if the <think> tags are mismatched.
         const maxAttempts = 5;
         let attempt = 0;
         let finalRes = null;
@@ -48,14 +48,14 @@ export class Local {
             console.log(`Awaiting local response... (model: ${model}, attempt: ${attempt})`);
             let res = null;
             try {
-                res = await this.send(this.chat_endpoint, {
+                let apiResponse = await this.send(this.chat_endpoint, {
                     model: model,
                     messages: messages,
                     stream: false,
                     ...(this.params || {})
                 });
-                if (res) {
-                    res = res['message']['content'];
+                if (apiResponse) {
+                    res = apiResponse['message']['content'];
                 } else {
                     res = 'No response data.';
                 }
@@ -67,38 +67,33 @@ export class Local {
                     console.log(err);
                     res = 'My brain disconnected, try again.';
                 }
-
             }
 
-            // If the model name includes "deepseek-r1" or "Andy-3.5-reasoning", then handle the <think> block.
-                const hasOpenTag = res.includes("<think>");
-                const hasCloseTag = res.includes("</think>");
-
-                // If there's a partial mismatch, retry to get a complete response.
-                if ((hasOpenTag && !hasCloseTag)) {
-                    console.warn("Partial <think> block detected. Re-generating...");
-                    continue; 
-                }
-            
-                // If </think> is present but <think> is not, prepend <think>
-                if (hasCloseTag && !hasOpenTag) {
-                    res = '<think>' + res;
-                }
-                // Changed this so if the model reasons, using <think> and </think> but doesn't start the message with <think>, <think> ges prepended to the message so no error occur.
-            
-                // If both tags appear, remove them (and everything inside).
-                if (hasOpenTag && hasCloseTag) {
-                    res = res.replace(/<think>[\s\S]*?<\/think>/g, '');
-                }
+            const hasOpenTag = res.includes("<think>");
+            const hasCloseTag = res.includes("</think>");
 
+            if ((hasOpenTag && !hasCloseTag)) {
+                console.warn("Partial <think> block detected. Re-generating...");
+                if (attempt < maxAttempts) continue;
+            }
+            if (hasCloseTag && !hasOpenTag) {
+                res = '<think>' + res;
+            }
+            if (hasOpenTag && hasCloseTag) {
+                res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+            }
             finalRes = res;
-            break; // Exit the loop if we got a valid response.
+            break;
         }
 
         if (finalRes == null) {
-            console.warn("Could not get a valid <think> block or normal response after max attempts.");
+            console.warn("Could not get a valid response after max attempts.");
             finalRes = 'I thought too hard, sorry, try again.';
         }
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/mistral.js

@@ -1,19 +1,17 @@
 import { Mistral as MistralClient } from '@mistralai/mistralai';
 import { getKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Mistral {
     #client;
-
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
 
         if (typeof url === "string") {
             console.warn("Mistral does not support custom URL's, ignoring!");
-
         }
-
         if (!getKey("MISTRAL_API_KEY")) {
             throw new Error("Mistral API Key missing, make sure to set MISTRAL_API_KEY in settings.json")
         }
@@ -26,13 +24,8 @@ export class Mistral {
         this.supportsRawImageInput = false; // Standard chat completions may not support raw images for all models.
 
         
-        // Prevents the following code from running when model not specified
-        if (typeof this.model_name === "undefined") return;
-
-        // get the model name without the "mistral" or "mistralai" prefix
-        // e.g "mistral/mistral-large-latest" -> "mistral-large-latest"
-        if (typeof model_name.split("/")[1] !== "undefined") {
-            this.model_name = model_name.split("/")[1];
+        if (typeof this.model_name === "string" && typeof this.model_name.split("/")[1] !== "undefined") {
+            this.model_name = this.model_name.split("/")[1];
         }
     }
 
@@ -43,22 +36,16 @@ export class Mistral {
         }
 
         let result;
-
+        const model = this.model_name || "mistral-large-latest";
+        const messages = [{ role: "system", content: systemMessage }];
+        messages.push(...strictFormat(turns));
         try {
-            const model = this.model_name || "mistral-large-latest";
-
-            const messages = [
-                { role: "system", content: systemMessage }
-            ];
-            messages.push(...strictFormat(turns));
-
             console.log('Awaiting mistral api response...')
             const response  = await this.#client.chat.complete({
                 model,
                 messages,
                 ...(this.params || {})
             });
-
             result = response.choices[0].message.content;
         } catch (err) {
             if (err.message.includes("A request containing images has been given to a model which does not have the 'vision' capability.")) {
@@ -68,24 +55,28 @@ export class Mistral {
             }
             console.log(err);
         }
-
+        if (typeof result === 'string') {
+            result = result.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), result);
         return result;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
-        imageMessages.push({
-            role: "user",
-            content: [
-                { type: "text", text: systemMessage },
-                {
-                    type: "image_url",
-                    imageUrl: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                }
-            ]
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
+        const imageFormattedTurns = [...original_turns];
+        const userMessageContent = [{ type: "text", text: systemMessage }];
+        userMessageContent.push({
+            type: "image_url",
+            imageUrl: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
         });
+        imageFormattedTurns.push({ role: "user", content: userMessageContent });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {

src/models/novita.js

@@ -1,6 +1,7 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 // llama, mistral
 export class Novita {
@@ -54,17 +55,26 @@ export class Novita {
               res = 'My brain disconnected, try again.';
           }
       }
-      if (res.includes('<think>')) {
-        let start = res.indexOf('<think>');
-        let end = res.indexOf('</think>') + 8;
-        if (start != -1) {
-          if (end != -1) {
-            res = res.substring(0, start) + res.substring(end);
-          } else {
-            res = res.substring(0, start+7);
+      if (typeof res === 'string') {
+          res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+      }
+      log(JSON.stringify(messages), res); // Log transformed res
+
+      // Existing stripping logic for <think> tags
+      if (res && typeof res === 'string' && res.includes('<think>')) {
+          let start = res.indexOf('<think>');
+          let end = res.indexOf('</think>') + 8; // length of '</think>'
+          if (start !== -1) { // Ensure '<think>' was found
+              if (end !== -1 && end > start + 7) { // Ensure '</think>' was found and is after '<think>'
+                  res = res.substring(0, start) + res.substring(end);
+              } else {
+                  // Malformed or missing end tag, strip from '<think>' onwards or handle as error
+                  // Original code: res = res.substring(0, start+7); This would leave "<think>"
+                  // Let's assume we strip from start if end is not valid.
+                  res = res.substring(0, start);
+              }
           }
-        }
-        res = res.trim();
+          res = res.trim();
       }
       return res;
   }

src/models/openrouter.js

@@ -1,22 +1,18 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class OpenRouter {
     constructor(model_name, url) {
         this.model_name = model_name;
-
         let config = {};
         config.baseURL = url || 'https://openrouter.ai/api/v1';
-
         const apiKey = getKey('OPENROUTER_API_KEY');
         if (!apiKey) {
             console.error('Error: OPENROUTER_API_KEY not found. Make sure it is set properly.');
         }
-
-        // Pass the API key to OpenAI compatible Api
-        config.apiKey = apiKey; 
-
+        config.apiKey = apiKey;
         this.openai = new OpenAIApi(config);
         // OpenRouter is a router; individual models might support vision.
         // This generic sendRequest does not format for vision. Use sendVisionRequest or specific model logic.
@@ -30,32 +26,79 @@ export class OpenRouter {
         let messages = [{ role: 'system', content: systemMessage }, ...turns];
         messages = strictFormat(messages);
 
-        // Choose a valid model from openrouter.ai (for example, "openai/gpt-4o")
         const pack = {
             model: this.model_name,
             messages,
-            stop: stop_seq
+            include_reasoning: true,
+            // stop: stop_seq // Commented out since some API providers on Openrouter do not support a stop sequence, such as Grok 3
         };
 
-        let res = null;
-        try {
-            console.log('Awaiting openrouter api response...');
-            let completion = await this.openai.chat.completions.create(pack);
-            if (!completion?.choices?.[0]) {
-                console.error('No completion or choices returned:', completion);
-                return 'No response received.';
+        const maxAttempts = 5;
+        let attempt = 0;
+        let finalRes = null;
+
+        while (attempt < maxAttempts) {
+            attempt++;
+            console.info(`Awaiting openrouter API response... (attempt: ${attempt})`);
+            let res = null;
+            try {
+                let completion = await this.openai.chat.completions.create(pack);
+                if (!completion?.choices?.[0]) {
+                    console.error('No completion or choices returned:', completion);
+                    return 'No response received.';
+                }
+
+                const logMessages = [{ role: "system", content: processedSystemMessage }].concat(turns);
+
+                if (completion.choices[0].finish_reason === 'length') {
+                    throw new Error('Context length exceeded');
+                }
+                
+                if (completion.choices[0].message.reasoning) {
+                    try{
+                        const reasoning = '<think>\n' + completion.choices[0].message.reasoning + '</think>\n';
+                        const content = completion.choices[0].message.content;
+
+                        // --- VISION LOGGING ---
+                        if (visionImageBuffer) {
+                            logVision(turns, visionImageBuffer, reasoning + "\n" + content, visionMessage);
+                        } else {
+                            log(JSON.stringify(logMessages), reasoning + "\n" + content);
+                        }
+                        res = content;
+                    } catch {}
+                } else {
+                    try {
+                        res = completion.choices[0].message.content;
+                        if (visionImageBuffer) {
+                            logVision(turns, visionImageBuffer, res, visionMessage);
+                        } else {
+                            log(JSON.stringify(logMessages), res);
+                        }
+                    } catch {
+                        console.warn("Unable to log due to unknown error!");
+                    }
+                }
+                // Trim <think> blocks from the final response if present.
+                if (res && res.includes("<think>") && res.includes("</think>")) {
+                    res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+                }
+
+                console.info('Received.');
+            } catch (err) {
+                console.error('Error while awaiting response:', err);
+                res = 'My brain disconnected, try again.';
             }
-            if (completion.choices[0].finish_reason === 'length') {
-                throw new Error('Context length exceeded');
-            }
-            console.log('Received.');
-            res = completion.choices[0].message.content;
-        } catch (err) {
-            console.error('Error while awaiting response:', err);
-            // If the error indicates a context-length problem, we can slice the turns array, etc.
-            res = 'My brain disconnected, try again.';
+
+            finalRes = res;
+            break; // Exit loop once a valid response is obtained.
         }
-        return res;
+
+        if (finalRes == null) {
+            console.warn("Could not get a valid <think> block or normal response after max attempts.");
+            finalRes = 'I thought too hard, sorry, try again.';
+        }
+        return finalRes;
     }
 
     async sendVisionRequest(messages, systemMessage, imageBuffer) {
@@ -76,9 +119,10 @@ export class OpenRouter {
         // sendVisionRequest formats its own message array; sendRequest here should not process new imageData.
         // Pass systemMessage and stop_seq as originally intended by sendRequest.
         return this.sendRequest(imageMessages, systemMessage, null, stop_seq);
+
     }
 
     async embed(text) {
         throw new Error('Embeddings are not supported by Openrouter.');
     }
-}
+}

src/models/prompter.js

@@ -465,8 +465,26 @@ export class Prompter {
     }
 
     async _saveLog(prompt, messages, generation, tag) {
-        if (!settings.log_all_prompts)
-            return;
+        // NEW LOGIC STARTS
+        switch (tag) {
+            case 'conversation':
+            case 'coding': // Assuming coding logs fall under normal data
+            case 'memSaving':
+                if (!settings.log_normal_data) return;
+                break;
+            // Add case for 'vision' if prompter.js starts logging vision prompts/responses via _saveLog
+            // case 'vision':
+            //     if (!settings.log_vision_data) return;
+            //     break;
+            default:
+                // If it's an unknown tag, perhaps log it if general logging is on, or ignore.
+                // For safety, let's assume if it's not specified, it doesn't get logged unless a general flag is on.
+                // However, the goal is to use specific flags. So, if a new tag appears, this logic should be updated.
+                // For now, if it doesn't match known tags that map to a setting, it won't log.
+                return;
+        }
+        // NEW LOGIC ENDS
+
         const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
         let logEntry;
         let task_id = this.agent.task.task_id;

src/models/qwen.js

@@ -1,6 +1,7 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Qwen {
     constructor(model_name, url, params) {
@@ -81,6 +82,10 @@ export class Qwen {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 

src/models/replicate.js

@@ -1,6 +1,7 @@
 import Replicate from 'replicate';
 import { toSinglePrompt } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // llama, mistral
 export class ReplicateAPI {
@@ -29,6 +30,7 @@ export class ReplicateAPI {
 		const prompt = toSinglePrompt(turns, null, stop_seq);
 		let model_name = this.model_name || 'meta/meta-llama-3-70b-instruct';
 
+		const logInputMessages = [{role: 'system', content: systemMessage}, ...turns];
 		const input = { 
 			prompt, 
 			system_prompt: systemMessage,
@@ -51,6 +53,10 @@ export class ReplicateAPI {
 			console.log(err);
 			res = 'My brain disconnected, try again.';
 		}
+		if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+		log(JSON.stringify(logInputMessages), res);
 		console.log('Received.');
 		return res;
 	}

src/models/vllm.js

@@ -1,9 +1,13 @@
 // This code uses Dashscope and HTTP to ensure the latest support for the Qwen model.
 // Qwen is also compatible with the OpenAI API format;
 
+// This code uses Dashscope and HTTP to ensure the latest support for the Qwen model.
+// Qwen is also compatible with the OpenAI API format;
+
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class VLLM {
     constructor(model_name, url) {
@@ -59,6 +63,10 @@ export class VLLM {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }