Merge pull request #5 from Sweaterdog/advanced-logging

Advanced logging

logger.js

@@ -0,0 +1,401 @@
+// --- START OF FILE logger.js ---
+
+import { writeFileSync, mkdirSync, existsSync, appendFileSync, readFileSync } from 'fs';
+import { join } from 'path';
+import settings from './settings.js'; // Import settings
+import path from 'path'; // Needed for path operations
+
+// --- Configuration ---
+const LOGS_DIR = './logs';
+const VISION_DATASET_DIR = join(LOGS_DIR, 'vision_dataset'); // HuggingFace dataset format
+const VISION_IMAGES_DIR = join(VISION_DATASET_DIR, 'images'); // Images subdirectory
+
+// --- Log File Paths ---
+const REASONING_LOG_FILE = join(LOGS_DIR, 'reasoning_logs.csv');
+const NORMAL_LOG_FILE = join(LOGS_DIR, 'normal_logs.csv');
+const VISION_METADATA_FILE = join(VISION_DATASET_DIR, 'metadata.jsonl'); // HF metadata format
+
+// --- Log Headers ---
+const TEXT_LOG_HEADER = 'input,output\n';
+
+// --- Log Counters ---
+let logCounts = {
+    normal: 0,
+    reasoning: 0,
+    vision: 0,
+    total: 0,
+    skipped_disabled: 0,
+    skipped_empty: 0,
+    vision_images_saved: 0,
+};
+
+// --- Helper Functions ---
+function ensureDirectoryExistence(dirPath) {
+    if (!existsSync(dirPath)) {
+        try {
+            mkdirSync(dirPath, { recursive: true });
+            console.log(`[Logger] Created directory: ${dirPath}`);
+        } catch (error) {
+            console.error(`[Logger] Error creating directory ${dirPath}:`, error);
+            return false;
+        }
+    }
+    return true;
+}
+
+function countLogEntries(logFile) {
+    if (!existsSync(logFile)) return 0;
+    try {
+        const data = readFileSync(logFile, 'utf8');
+        const lines = data.split('\n').filter(line => line.trim());
+        // Check if the first line looks like a header before subtracting
+        const hasHeader = lines.length > 0 && lines[0].includes(',');
+        return Math.max(0, hasHeader ? lines.length - 1 : lines.length);
+    } catch (err) {
+        console.error(`[Logger] Error reading log file ${logFile}:`, err);
+        return 0;
+    }
+}
+
+
+function ensureLogFile(logFile, header) {
+     if (!ensureDirectoryExistence(path.dirname(logFile))) return false; // Ensure parent dir exists
+
+     if (!existsSync(logFile)) {
+        try {
+            writeFileSync(logFile, header);
+            console.log(`[Logger] Created log file: ${logFile}`);
+        } catch (error) {
+            console.error(`[Logger] Error creating log file ${logFile}:`, error);
+            return false;
+        }
+    } else {
+         try {
+            const content = readFileSync(logFile, 'utf-8');
+            const headerLine = header.split('\n')[0];
+            // If file is empty or header doesn't match, overwrite/create header
+            if (!content.trim() || !content.startsWith(headerLine)) {
+                 // Attempt to prepend header if file has content but wrong/no header
+                 if(content.trim() && !content.startsWith(headerLine)) {
+                    console.warn(`[Logger] Log file ${logFile} seems to be missing or has an incorrect header. Prepending correct header.`);
+                    writeFileSync(logFile, header + content);
+                 } else {
+                    // File is empty or correctly headed, just ensure header is there
+                     writeFileSync(logFile, header);
+                 }
+                 console.log(`[Logger] Ensured header in log file: ${logFile}`);
+            }
+        } catch (error) {
+            console.error(`[Logger] Error checking/writing header for log file ${logFile}:`, error);
+            // Proceed cautiously, maybe log an error and continue?
+        }
+    }
+    return true;
+}
+
+
+function writeToLogFile(logFile, csvEntry) {
+    try {
+        appendFileSync(logFile, csvEntry);
+        // console.log(`[Logger] Logged data to ${logFile}`); // Keep console less noisy
+    } catch (error) {
+        console.error(`[Logger] Error writing to CSV log file ${logFile}:`, error);
+    }
+}
+
+// --- Auto-Detection for Log Type (Based on Response Content) ---
+function determineLogType(response) {
+    // Reasoning check: needs <think>...</think> but ignore the specific 'undefined' placeholder
+    const isReasoning = response.includes('<think>') && response.includes('</think>') && !response.includes('<think>\nundefined</think>');
+
+    if (isReasoning) {
+        return 'reasoning';
+    } else {
+        return 'normal';
+    }
+}
+
+function sanitizeForCsv(value) {
+    if (typeof value !== 'string') {
+        value = String(value);
+    }
+    // Escape double quotes by doubling them and enclose the whole string in double quotes
+    return `"${value.replace(/"/g, '""')}"`;
+}
+
+// Helper function to clean reasoning markers from input
+function cleanReasoningMarkers(input) {
+    if (typeof input !== 'string') {
+        return input;
+    }
+    
+    // Remove /think and /no_think markers
+    return input.replace(/\/think/g, '').replace(/\/no_think/g, '').trim();
+}
+
+// --- Main Logging Function (for text-based input/output) ---
+export function log(input, response) {
+    const trimmedInputStr = input ? (typeof input === 'string' ? input.trim() : JSON.stringify(input)) : "";
+    const trimmedResponse = response ? String(response).trim() : ""; // Ensure response is a string
+
+    // Clean reasoning markers from input before logging
+    const cleanedInput = cleanReasoningMarkers(trimmedInputStr);
+
+    // Basic filtering
+    if (!cleanedInput && !trimmedResponse) {
+        logCounts.skipped_empty++;
+        return;
+    }
+    if (cleanedInput === trimmedResponse) {
+         logCounts.skipped_empty++;
+        return;
+    }
+     // Avoid logging common error messages that aren't useful training data
+    const errorMessages = [
+        "My brain disconnected, try again.",
+        "My brain just kinda stopped working. Try again.",
+        "I thought too hard, sorry, try again.",
+        "*no response*",
+        "No response received.",
+        "No response data.",
+        "Failed to send", // Broader match
+        "Error:", // Broader match
+        "Vision is only supported",
+        "Context length exceeded",
+        "Image input modality is not enabled",
+        "An unexpected error occurred",
+        // Add more generic errors/placeholders as needed
+    ];
+    // Also check for responses that are just the input repeated (sometimes happens with errors)
+    if (errorMessages.some(err => trimmedResponse.includes(err)) || trimmedResponse === cleanedInput) {
+        logCounts.skipped_empty++;
+        // console.warn(`[Logger] Skipping log due to error/placeholder/repeat: "${trimmedResponse.substring(0, 70)}..."`);
+        return;
+    }
+
+
+    const logType = determineLogType(trimmedResponse);
+    let logFile;
+    let header;
+    let settingFlag;
+
+    switch (logType) {
+        case 'reasoning':
+            logFile = REASONING_LOG_FILE;
+            header = TEXT_LOG_HEADER;
+            settingFlag = settings.log_reasoning_data;
+            break;
+        case 'normal':
+        default:
+            logFile = NORMAL_LOG_FILE;
+            header = TEXT_LOG_HEADER;
+            settingFlag = settings.log_normal_data;
+            break;
+    }
+
+    // Check if logging for this type is enabled
+    if (!settingFlag) {
+        logCounts.skipped_disabled++;
+        return;
+    }
+
+    // Ensure directory and file exist
+    if (!ensureLogFile(logFile, header)) return; // ensureLogFile now checks parent dir too
+
+    // Prepare the CSV entry using the sanitizer with cleaned input
+    const safeInput = sanitizeForCsv(cleanedInput);
+    const safeResponse = sanitizeForCsv(trimmedResponse);
+    const csvEntry = `${safeInput},${safeResponse}\n`;
+
+    // Write to the determined log file
+    writeToLogFile(logFile, csvEntry);
+
+    // Update counts
+    logCounts[logType]++;
+    logCounts.total++; // Total here refers to text logs primarily
+
+    // Display summary periodically (based on total text logs)
+    if (logCounts.normal + logCounts.reasoning > 0 && (logCounts.normal + logCounts.reasoning) % 20 === 0) {
+       printSummary();
+    }
+}
+
+// --- Enhanced Vision Logging Function for HuggingFace Dataset Format ---
+export function logVision(conversationHistory, imageBuffer, response, visionMessage = null) {
+    if (!settings.log_vision_data) {
+        logCounts.skipped_disabled++;
+        return;
+    }
+
+    const trimmedResponse = response ? String(response).trim() : "";
+    
+    if (!conversationHistory || conversationHistory.length === 0 || !trimmedResponse || !imageBuffer) {
+        logCounts.skipped_empty++;
+        return;
+    }
+
+    // Filter out error messages
+    const errorMessages = [
+        "My brain disconnected, try again.",
+        "My brain just kinda stopped working. Try again.",
+        "I thought too hard, sorry, try again.",
+        "*no response*",
+        "No response received.",
+        "No response data.",
+        "Failed to send",
+        "Error:",
+        "Vision is only supported",
+        "Context length exceeded",
+        "Image input modality is not enabled",
+        "An unexpected error occurred",
+    ];
+    
+    if (errorMessages.some(err => trimmedResponse.includes(err))) {
+        logCounts.skipped_empty++;
+        return;
+    }
+
+    // Ensure directories exist
+    if (!ensureDirectoryExistence(VISION_DATASET_DIR)) return;
+    if (!ensureDirectoryExistence(VISION_IMAGES_DIR)) return;
+
+    try {
+        // Generate unique filename for the image
+        const timestamp = Date.now();
+        const randomSuffix = Math.random().toString(36).substring(2, 8);
+        const imageFilename = `vision_${timestamp}_${randomSuffix}.jpg`;
+        const imagePath = join(VISION_IMAGES_DIR, imageFilename);
+        const relativeImagePath = `images/${imageFilename}`; // Relative path for metadata
+
+        // Save the image
+        writeFileSync(imagePath, imageBuffer);
+        logCounts.vision_images_saved++;
+
+        // Extract the actual message sent with the image
+        // This is typically the vision prompt/instruction
+        let inputMessage = visionMessage;
+        if (!inputMessage && conversationHistory.length > 0) {
+            // Try to get the last user message or system message
+            const lastMessage = conversationHistory[conversationHistory.length - 1];
+            if (typeof lastMessage.content === 'string') {
+                inputMessage = lastMessage.content;
+            } else if (Array.isArray(lastMessage.content)) {
+                // Find text content in the message
+                const textContent = lastMessage.content.find(c => c.type === 'text');
+                inputMessage = textContent ? textContent.text : '';
+            }
+        }
+
+        // Fallback to conversation history if no specific message
+        if (!inputMessage) {
+            inputMessage = formatConversationInput(conversationHistory);
+        }
+
+        // Create metadata entry in JSONL format for HuggingFace
+        const metadataEntry = {
+            file_name: relativeImagePath,
+            text: inputMessage,
+            response: trimmedResponse,
+            timestamp: timestamp
+        };
+
+        // Append to metadata JSONL file
+        const jsonlLine = JSON.stringify(metadataEntry) + '\n';
+        appendFileSync(VISION_METADATA_FILE, jsonlLine);
+        
+        logCounts.vision++;
+        logCounts.total++;
+
+        // Display summary periodically
+        if (logCounts.vision > 0 && logCounts.vision % 10 === 0) {
+            printSummary();
+        }
+
+    } catch (error) {
+        console.error(`[Logger] Error logging vision data:`, error);
+    }
+}
+
+// Helper function to format conversation history as fallback
+function formatConversationInput(conversationHistory) {
+    if (!conversationHistory || conversationHistory.length === 0) return '';
+    
+    const formattedHistory = [];
+    
+    for (const turn of conversationHistory) {
+        const formattedTurn = {
+            role: turn.role || 'user',
+            content: []
+        };
+
+        // Handle different content formats
+        if (typeof turn.content === 'string') {
+            formattedTurn.content.push({
+                type: 'text',
+                text: turn.content
+            });
+        } else if (Array.isArray(turn.content)) {
+            // Already in the correct format
+            formattedTurn.content = turn.content;
+        } else if (turn.content && typeof turn.content === 'object') {
+            // Convert object to array format
+            if (turn.content.text) {
+                formattedTurn.content.push({
+                    type: 'text',
+                    text: turn.content.text
+                });
+            }
+            if (turn.content.image) {
+                formattedTurn.content.push({
+                    type: 'image',
+                    image: turn.content.image
+                });
+            }
+        }
+
+        formattedHistory.push(formattedTurn);
+    }
+    
+    return JSON.stringify(formattedHistory);
+}
+
+function printSummary() {
+    const totalStored = logCounts.normal + logCounts.reasoning + logCounts.vision;
+    console.log('\n' + '='.repeat(60));
+    console.log('LOGGER SUMMARY');
+    console.log('-'.repeat(60));
+    console.log(`Normal logs stored:    ${logCounts.normal}`);
+    console.log(`Reasoning logs stored: ${logCounts.reasoning}`);
+    console.log(`Vision logs stored:    ${logCounts.vision} (Images saved: ${logCounts.vision_images_saved})`);
+    console.log(`Skipped (disabled):    ${logCounts.skipped_disabled}`);
+    console.log(`Skipped (empty/err):   ${logCounts.skipped_empty}`);
+    console.log('-'.repeat(60));
+    console.log(`Total logs stored:     ${totalStored}`);
+    console.log('='.repeat(60) + '\n');
+}
+
+// Initialize counts at startup
+function initializeCounts() {
+    logCounts.normal = countLogEntries(NORMAL_LOG_FILE);
+    logCounts.reasoning = countLogEntries(REASONING_LOG_FILE);
+    logCounts.vision = countVisionEntries(VISION_METADATA_FILE);
+    // Total count will be accumulated during runtime
+    console.log(`[Logger] Initialized log counts: Normal=${logCounts.normal}, Reasoning=${logCounts.reasoning}, Vision=${logCounts.vision}`);
+}
+
+function countVisionEntries(metadataFile) {
+    if (!existsSync(metadataFile)) return 0;
+    try {
+        const data = readFileSync(metadataFile, 'utf8');
+        const lines = data.split('\n').filter(line => line.trim());
+        return lines.length;
+    } catch (err) {
+        console.error(`[Logger] Error reading vision metadata file ${metadataFile}:`, err);
+        return 0;
+    }
+}
+
+// Initialize counts at startup
+initializeCounts();
+
+// --- END OF FILE logger.js ---

settings.js

@@ -44,7 +44,11 @@ const settings = {
     "verbose_commands": true, // show full command syntax
     "narrate_behavior": true, // chat simple automatic actions ('Picking up item!')
     "chat_bot_messages": true, // publicly chat messages to other bots
-    "log_all_prompts": false, // log ALL prompts to file
+    
+    "log_normal_data": false,
+    "log_reasoning_data": false,
+    "log_vision_data": false,
+    
 }
 
 // these environment variables override certain settings
@@ -69,8 +73,5 @@ if (process.env.MAX_MESSAGES) {
 if (process.env.NUM_EXAMPLES) {
     settings.num_examples = process.env.NUM_EXAMPLES;
 }
-if (process.env.LOG_ALL) {
-    settings.log_all_prompts = process.env.LOG_ALL;
-}
 
 export default settings;

src/models/claude.js

@@ -1,18 +1,16 @@
 import Anthropic from '@anthropic-ai/sdk';
 import { strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Claude {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params || {};
-
         let config = {};
         if (url)
             config.baseURL = url;
-        
         config.apiKey = getKey('ANTHROPIC_API_KEY');
-
         this.anthropic = new Anthropic(config);
     }
 
@@ -23,8 +21,7 @@ export class Claude {
             console.log('Awaiting anthropic api response...')
             if (!this.params.max_tokens) {
                 if (this.params.thinking?.budget_tokens) {
-                    this.params.max_tokens = this.params.thinking.budget_tokens + 1000;
+                    this.params.max_tokens = this.params.thinking.budget_tokens + 1000; // max_tokens must be greater
-                    // max_tokens must be greater than thinking.budget_tokens
                 } else {
                     this.params.max_tokens = 4096;
                 }
@@ -35,9 +32,7 @@ export class Claude {
                 messages: messages,
                 ...(this.params || {})
             });
-
             console.log('Received.')
-            // get first content of type text
             const textContent = resp.content.find(content => content.type === 'text');
             if (textContent) {
                 res = textContent.text;
@@ -45,8 +40,7 @@ export class Claude {
                 console.warn('No text content found in the response.');
                 res = 'No response from Claude.';
             }
-        }
+        } catch (err) {
-        catch (err) {
             if (err.message.includes("does not support image input")) {
                 res = "Vision is only supported by certain models.";
             } else {
@@ -54,18 +48,17 @@ export class Claude {
             }
             console.log(err);
         }
+        const logMessagesForClaude = [{ role: "system", content: systemMessage }].concat(turns);
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(logMessagesForClaude), res);
         return res;
     }
 
     async sendVisionRequest(turns, systemMessage, imageBuffer) {
-        const imageMessages = [...turns];
+        const visionUserMessageContent = [
-        imageMessages.push({
+            { type: "text", text: systemMessage },
-            role: "user",
-            content: [
-                {
-                    type: "text",
-                    text: systemMessage
-                },
             {
                 type: "image",
                 source: {
@@ -74,10 +67,15 @@ export class Claude {
                     data: imageBuffer.toString('base64')
                 }
             }
-            ]
+        ];
-        });
+        const turnsForAPIRequest = [...turns, { role: "user", content: visionUserMessageContent }];
 
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(turnsForAPIRequest, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {

src/models/deepseek.js

@@ -1,43 +1,36 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class DeepSeek {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
-
         let config = {};
-
         config.baseURL = url || 'https://api.deepseek.com';
         config.apiKey = getKey('DEEPSEEK_API_KEY');
-
         this.openai = new OpenAIApi(config);
     }
 
     async sendRequest(turns, systemMessage, stop_seq='***') {
         let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
-
         messages = strictFormat(messages);
-
         const pack = {
             model: this.model_name || "deepseek-chat",
             messages,
             stop: stop_seq,
             ...(this.params || {})
         };
-
         let res = null;
         try {
             console.log('Awaiting deepseek api response...')
-            // console.log('Messages:', messages);
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded'); 
             console.log('Received.')
             res = completion.choices[0].message.content;
-        }
+        } catch (err) {
-        catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
@@ -46,6 +39,10 @@ export class DeepSeek {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 
@@ -53,6 +50,3 @@ export class DeepSeek {
         throw new Error('Embeddings are not supported by Deepseek.');
     }
 }
-
-
-

src/models/gemini.js

@@ -1,6 +1,7 @@
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import { toSinglePrompt, strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Gemini {
     constructor(model_name, url, params) {
@@ -8,28 +9,12 @@ export class Gemini {
         this.params = params;
         this.url = url;
         this.safetySettings = [
-            {
+            { "category": "HARM_CATEGORY_DANGEROUS", "threshold": "BLOCK_NONE" },
-                "category": "HARM_CATEGORY_DANGEROUS",
+            { "category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE" },
-                "threshold": "BLOCK_NONE",
+            { "category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE" },
-            },
+            { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE" },
-            {
+            { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE" },
-                "category": "HARM_CATEGORY_HARASSMENT",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_HATE_SPEECH",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
-                "threshold": "BLOCK_NONE",
-            },
-            {
-                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
-                "threshold": "BLOCK_NONE",
-            },
         ];
-
         this.genAI = new GoogleGenerativeAI(getKey('GEMINI_API_KEY'));
     }
 
@@ -40,20 +25,12 @@ export class Gemini {
             // systemInstruction does not work bc google is trash
         };
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel(modelConfig, { baseUrl: this.url }, { safetySettings: this.safetySettings });
-                modelConfig,
-                { baseUrl: this.url },
-                { safetySettings: this.safetySettings }
-            );
         } else {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel(modelConfig, { safetySettings: this.safetySettings });
-                modelConfig,
-                { safetySettings: this.safetySettings }
-            );
         }
-
         console.log('Awaiting Google API response...');
-
+        const originalTurnsForLog = [{role: 'system', content: systemMessage}, ...turns];
         turns.unshift({ role: 'system', content: systemMessage });
         turns = strictFormat(turns);
         let contents = [];
@@ -63,25 +40,14 @@ export class Gemini {
                 parts: [{ text: turn.content }]
             });
         }
-
         const result = await model.generateContent({
             contents,
-            generationConfig: {
+            generationConfig: { ...(this.params || {}) }
-                ...(this.params || {})
-            }
         });
         const response = await result.response;
         let text;
-
-        // Handle "thinking" models since they smart 
         if (this.model_name && this.model_name.includes("thinking")) {
-            if (
+            if (response.candidates?.length > 0 && response.candidates[0].content?.parts?.length > 1) {
-                response.candidates &&
-                response.candidates.length > 0 &&
-                response.candidates[0].content &&
-                response.candidates[0].content.parts &&
-                response.candidates[0].content.parts.length > 1
-            ) {
                 text = response.candidates[0].content.parts[1].text;
             } else {
                 console.warn("Unexpected response structure for thinking model:", response);
@@ -90,34 +56,22 @@ export class Gemini {
         } else {
             text = response.text();
         }
-
         console.log('Received.');
-
+        if (typeof text === 'string') {
+            text = text.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(originalTurnsForLog), text);
         return text;
     }
 
     async sendVisionRequest(turns, systemMessage, imageBuffer) {
         let model;
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel({ model: this.model_name || "gemini-1.5-flash" }, { baseUrl: this.url }, { safetySettings: this.safetySettings });
-                { model: this.model_name || "gemini-1.5-flash" },
-                { baseUrl: this.url },
-                { safetySettings: this.safetySettings }
-            );
         } else {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel({ model: this.model_name || "gemini-1.5-flash" }, { safetySettings: this.safetySettings });
-                { model: this.model_name || "gemini-1.5-flash" },
-                { safetySettings: this.safetySettings }
-            );
         }
-
+        const imagePart = { inlineData: { data: imageBuffer.toString('base64'), mimeType: 'image/jpeg' } };
-        const imagePart = {
-            inlineData: {
-                data: imageBuffer.toString('base64'),
-                mimeType: 'image/jpeg'
-            }
-        };
-
         const stop_seq = '***';
         const prompt = toSinglePrompt(turns, systemMessage, stop_seq, 'model');
         let res = null;
@@ -127,6 +81,9 @@ export class Gemini {
             const response = await result.response;
             const text = response.text();
             console.log('Received.');
+            if (imageBuffer && text) {
+                logVision(turns, imageBuffer, text, prompt);
+            }
             if (!text.includes(stop_seq)) return text;
             const idx = text.indexOf(stop_seq);
             res = text.slice(0, idx);
@@ -137,6 +94,11 @@ export class Gemini {
             } else {
                 res = "An unexpected error occurred, please try again.";
             }
+            const loggedTurnsForError = [{role: 'system', content: systemMessage}, ...turns];
+            if (typeof res === 'string') {
+                res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+            }
+            log(JSON.stringify(loggedTurnsForError), res);
         }
         return res;
     }
@@ -144,16 +106,10 @@ export class Gemini {
     async embed(text) {
         let model;
         if (this.url) {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel({ model: "text-embedding-004" }, { baseUrl: this.url });
-                { model: "text-embedding-004" },
-                { baseUrl: this.url }
-            );
         } else {
-            model = this.genAI.getGenerativeModel(
+            model = this.genAI.getGenerativeModel({ model: "text-embedding-004" });
-                { model: "text-embedding-004" }
-            );
         }
-
         const result = await model.embedContent(text);
         return result.embedding.values;
     }

src/models/glhf.js

@@ -1,5 +1,6 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class GLHF {
     constructor(model_name, url) {
@@ -15,7 +16,6 @@ export class GLHF {
     }
 
     async sendRequest(turns, systemMessage, stop_seq = '***') {
-        // Construct the message array for the API request.
         let messages = [{ role: 'system', content: systemMessage }].concat(turns);
         const pack = {
             model: this.model_name || "hf:meta-llama/Llama-3.1-405B-Instruct",
@@ -36,17 +36,15 @@ export class GLHF {
                     throw new Error('Context length exceeded');
                 }
                 let res = completion.choices[0].message.content;
-                // If there's an open <think> tag without a corresponding </think>, retry.
                 if (res.includes("<think>") && !res.includes("</think>")) {
                     console.warn("Partial <think> block detected. Re-generating...");
-                    continue;
+                    if (attempt < maxAttempts) continue;
                 }
-                // If there's a closing </think> tag but no opening <think>, prepend one.
                 if (res.includes("</think>") && !res.includes("<think>")) {
                     res = "<think>" + res;
                 }
                 finalRes = res.replace(/<\|separator\|>/g, '*no response*');
-                break; // Valid response obtained.
+                break;
             } catch (err) {
                 if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) {
                     console.log('Context length exceeded, trying again with shorter context.');
@@ -61,6 +59,11 @@ export class GLHF {
         if (finalRes === null) {
             finalRes = "I thought too hard, sorry, try again";
         }
+
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/gpt.js

@@ -1,21 +1,18 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class GPT {
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
-
         let config = {};
         if (url)
             config.baseURL = url;
-
         if (hasKey('OPENAI_ORG_ID'))
             config.organization = getKey('OPENAI_ORG_ID');
-
         config.apiKey = getKey('OPENAI_API_KEY');
-
         this.openai = new OpenAIApi(config);
     }
 
@@ -31,19 +28,15 @@ export class GPT {
         if (this.model_name.includes('o1')) {
             delete pack.stop;
         }
-
         let res = null;
-
         try {
             console.log('Awaiting openai api response from model', this.model_name)
-            // console.log('Messages:', messages);
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded'); 
             console.log('Received.')
             res = completion.choices[0].message.content;
-        }
+        } catch (err) {
-        catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
@@ -55,25 +48,32 @@ export class GPT {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
+        const imageFormattedTurns = [...original_turns];
-        imageMessages.push({
+        imageFormattedTurns.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
                 }
             ]
         });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {
@@ -86,8 +86,4 @@ export class GPT {
         });
         return embedding.data[0].embedding;
     }
-
 }
-
-
-

src/models/grok.js

@@ -1,5 +1,6 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // xAI doesn't supply a SDK for their models, but fully supports OpenAI and Anthropic SDKs
 export class Grok {
@@ -7,39 +8,32 @@ export class Grok {
         this.model_name = model_name;
         this.url = url;
         this.params = params;
-
         let config = {};
         if (url)
             config.baseURL = url;
         else
             config.baseURL = "https://api.x.ai/v1"
-
         config.apiKey = getKey('XAI_API_KEY');
-
         this.openai = new OpenAIApi(config);
     }
 
     async sendRequest(turns, systemMessage, stop_seq='***') {
         let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
-
         const pack = {
             model: this.model_name || "grok-beta",
             messages,
             stop: [stop_seq],
             ...(this.params || {})
         };
-
         let res = null;
         try {
             console.log('Awaiting xai api response...')
-            ///console.log('Messages:', messages);
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')
                 throw new Error('Context length exceeded'); 
             console.log('Received.')
             res = completion.choices[0].message.content;
-        }
+        } catch (err) {
-        catch (err) {
             if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
                 console.log('Context length exceeded, trying again with shorter context.');
                 return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
@@ -52,31 +46,36 @@ export class Grok {
             }
         }
         // sometimes outputs special token <|separator|>, just replace it
-        return res.replace(/<\|separator\|>/g, '*no response*');
+        let finalResponseText = res ? res.replace(/<\|separator\|>/g, '*no response*') : (res === null ? "*no response*" : res);
+        if (typeof finalResponseText === 'string') {
+            finalResponseText = finalResponseText.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalResponseText);
+        return finalResponseText;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
+        const imageFormattedTurns = [...original_turns];
-        imageMessages.push({
+        imageFormattedTurns.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
                 }
             ]
         });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
     
     async embed(text) {
         throw new Error('Embeddings are not supported by Grok.');
     }
 }
-
-
-

src/models/groq.js

@@ -1,14 +1,13 @@
 import Groq from 'groq-sdk'
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // THIS API IS NOT TO BE CONFUSED WITH GROK!
 // Go to grok.js for that. :)
 
 // Umbrella class for everything under the sun... That GroqCloud provides, that is.
 export class GroqCloudAPI {
-
     constructor(model_name, url, params) {
-
         this.model_name = model_name;
         this.url = url;
         this.params = params || {};
@@ -18,21 +17,15 @@ export class GroqCloudAPI {
             delete this.params.tools;
         // This is just a bit of future-proofing in case we drag Mindcraft in that direction.
 
-        // I'm going to do a sneaky ReplicateAPI theft for a lot of this, aren't I?
         if (this.url)
             console.warn("Groq Cloud has no implementation for custom URLs. Ignoring provided URL.");
 
         this.groq = new Groq({ apiKey: getKey('GROQCLOUD_API_KEY') });
-
-
     }
 
     async sendRequest(turns, systemMessage, stop_seq = null) {
-        // Construct messages array
         let messages = [{"role": "system", "content": systemMessage}].concat(turns);
-
         let res = null;
-
         try {
             console.log("Awaiting Groq response...");
 
@@ -42,7 +35,6 @@ export class GroqCloudAPI {
                 this.params.max_completion_tokens = this.params.max_tokens;
                 delete this.params.max_tokens;
             }
-
             if (!this.params.max_completion_tokens) {
                 this.params.max_completion_tokens = 4000;
             }
@@ -55,11 +47,15 @@ export class GroqCloudAPI {
                 ...(this.params || {})
             });
 
-            res = completion.choices[0].message;
+            let responseText = completion.choices[0].message.content;
-
+            if (typeof responseText === 'string') {
-            res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+                responseText = responseText.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
             }
-        catch(err) {
+            log(JSON.stringify(messages), responseText);
+            // Original cleaning of <think> tags for the *returned* response (not affecting log)
+            responseText = responseText.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+            return responseText;
+        } catch(err) {
             if (err.message.includes("content must be a string")) {
                 res = "Vision is only supported by certain models.";
             } else {
@@ -67,26 +63,33 @@ export class GroqCloudAPI {
                 res = "My brain disconnected, try again.";
             }
             console.log(err);
+            if (typeof res === 'string') {
+                res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
             }
+            log(JSON.stringify(messages), res);
             return res;
         }
+    }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
-        const imageMessages = messages.filter(message => message.role !== 'system');
+        const imageMessages = [...original_turns];
         imageMessages.push({
             role: "user",
             content: [
                 { type: "text", text: systemMessage },
                 {
                     type: "image_url",
-                    image_url: {
+                    image_url: { url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}` }
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                    }
                 }
             ]
         });
         
-        return this.sendRequest(imageMessages);
+        const res = await this.sendRequest(imageMessages, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(_) {

src/models/huggingface.js

@@ -1,31 +1,26 @@
 import { toSinglePrompt } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
 import { HfInference } from "@huggingface/inference";
+import { log, logVision } from '../../logger.js';
 
 export class HuggingFace {
   constructor(model_name, url, params) {
-    // Remove 'huggingface/' prefix if present
     this.model_name = model_name.replace('huggingface/', '');
     this.url = url;
     this.params = params;
-
     if (this.url) {
       console.warn("Hugging Face doesn't support custom urls!");
     }
-
     this.huggingface = new HfInference(getKey('HUGGINGFACE_API_KEY'));
   }
 
   async sendRequest(turns, systemMessage) {
     const stop_seq = '***';
-    // Build a single prompt from the conversation turns
     const prompt = toSinglePrompt(turns, null, stop_seq);
-    // Fallback model if none was provided
     const model_name = this.model_name || 'meta-llama/Meta-Llama-3-8B';
-    // Combine system message with the prompt
+    const logInputMessages = [{role: 'system', content: systemMessage}, ...turns];
-    const input = systemMessage + "\n" + prompt;
+    const input = systemMessage + "
-
+" + prompt;
-    // We'll try up to 5 times in case of partial <think> blocks for DeepSeek-R1 models.
     const maxAttempts = 5;
     let attempt = 0;
     let finalRes = null;
@@ -35,7 +30,6 @@ export class HuggingFace {
       console.log(`Awaiting Hugging Face API response... (model: ${model_name}, attempt: ${attempt})`);
       let res = '';
       try {
-        // Consume the streaming response chunk by chunk
         for await (const chunk of this.huggingface.chatCompletionStream({
           model: model_name,
           messages: [{ role: "user", content: input }],
@@ -46,36 +40,32 @@ export class HuggingFace {
       } catch (err) {
         console.log(err);
         res = 'My brain disconnected, try again.';
-        // Break out immediately; we only retry when handling partial <think> tags.
         break;
       }
 
-      // If the model is DeepSeek-R1, check for mismatched <think> blocks.
       const hasOpenTag = res.includes("<think>");
       const hasCloseTag = res.includes("</think>");
 
-        // If there's a partial mismatch, warn and retry the entire request.
       if ((hasOpenTag && !hasCloseTag)) {
         console.warn("Partial <think> block detected. Re-generating...");
-          continue;
+        if (attempt < maxAttempts) continue;
       }
-
-        // If both tags are present, remove the <think> block entirely.
       if (hasOpenTag && hasCloseTag) {
         res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
       }
-
       finalRes = res;
-      break; // Exit loop if we got a valid response.
+      break;
     }
 
-    // If no valid response was obtained after max attempts, assign a fallback.
     if (finalRes == null) {
-      console.warn("Could not get a valid <think> block or normal response after max attempts.");
+      console.warn("Could not get a valid response after max attempts.");
       finalRes = 'I thought too hard, sorry, try again.';
     }
     console.log('Received.');
-    console.log(finalRes);
+    if (typeof finalRes === 'string') {
+        finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+    }
+    log(JSON.stringify(logInputMessages), finalRes);
     return finalRes;
   }
 

src/models/hyperbolic.js

@@ -1,30 +1,18 @@
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 export class Hyperbolic {
     constructor(modelName, apiUrl) {
         this.modelName = modelName || "deepseek-ai/DeepSeek-V3";
         this.apiUrl = apiUrl || "https://api.hyperbolic.xyz/v1/chat/completions";
-
-        // Retrieve the Hyperbolic API key from keys.js
         this.apiKey = getKey('HYPERBOLIC_API_KEY');
         if (!this.apiKey) {
             throw new Error('HYPERBOLIC_API_KEY not found. Check your keys.js file.');
         }
     }
 
-    /**
-     * Sends a chat completion request to the Hyperbolic endpoint.
-     *
-     * @param {Array} turns - An array of message objects, e.g. [{role: 'user', content: 'Hi'}].
-     * @param {string} systemMessage - The system prompt or instruction.
-     * @param {string} stopSeq - A stopping sequence, default '***'.
-     * @returns {Promise<string>} - The model's reply.
-     */
     async sendRequest(turns, systemMessage, stopSeq = '***') {
-        // Prepare the messages with a system prompt at the beginning
         const messages = [{ role: 'system', content: systemMessage }, ...turns];
-
-        // Build the request payload
         const payload = {
             model: this.modelName,
             messages: messages,
@@ -37,14 +25,11 @@ export class Hyperbolic {
         const maxAttempts = 5;
         let attempt = 0;
         let finalRes = null;
+        let rawCompletionContent = null;
 
         while (attempt < maxAttempts) {
             attempt++;
             console.log(`Awaiting Hyperbolic API response... (attempt: ${attempt})`);
-            console.log('Messages:', messages);
-
-            let completionContent = null;
-
             try {
                 const response = await fetch(this.apiUrl, {
                     method: 'POST',
@@ -54,56 +39,56 @@ export class Hyperbolic {
                     },
                     body: JSON.stringify(payload)
                 });
-
                 if (!response.ok) {
                     throw new Error(`HTTP error! status: ${response.status}`);
                 }
-
                 const data = await response.json();
                 if (data?.choices?.[0]?.finish_reason === 'length') {
                     throw new Error('Context length exceeded');
                 }
-
+                rawCompletionContent = data?.choices?.[0]?.message?.content || '';
-                completionContent = data?.choices?.[0]?.message?.content || '';
                 console.log('Received response from Hyperbolic.');
             } catch (err) {
-                if (
+                if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) {
-                    (err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') &&
-                    turns.length > 1
-                ) {
                     console.log('Context length exceeded, trying again with a shorter context...');
                     return await this.sendRequest(turns.slice(1), systemMessage, stopSeq);
                 } else {
                     console.error(err);
-                    completionContent = 'My brain disconnected, try again.';
+                    rawCompletionContent = 'My brain disconnected, try again.';
+                    finalRes = rawCompletionContent;
+                    break;
                 }
             }
 
-            // Check for <think> blocks
+            let processedContent = rawCompletionContent;
-            const hasOpenTag = completionContent.includes("<think>");
+            const hasOpenTag = processedContent.includes("<think>");
-            const hasCloseTag = completionContent.includes("</think>");
+            const hasCloseTag = processedContent.includes("</think>");
 
             if ((hasOpenTag && !hasCloseTag)) {
                 console.warn("Partial <think> block detected. Re-generating...");
-                continue; // Retry the request
+                if (attempt < maxAttempts) continue;
             }
-
             if (hasCloseTag && !hasOpenTag) {
-                completionContent = '<think>' + completionContent;
+                processedContent = '<think>' + processedContent;
             }
-
             if (hasOpenTag && hasCloseTag) {
-                completionContent = completionContent.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+                processedContent = processedContent.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+            }
+            finalRes = processedContent.replace(/<\|separator\|>/g, '*no response*');
+            if (!(hasOpenTag && !hasCloseTag && attempt < maxAttempts)) {
+                break;
             }
-
-            finalRes = completionContent.replace(/<\|separator\|>/g, '*no response*');
-            break; // Valid response obtained—exit loop
         }
 
         if (finalRes == null) {
-            console.warn("Could not get a valid <think> block or normal response after max attempts.");
+            finalRes = rawCompletionContent || 'I thought too hard, sorry, try again.';
-            finalRes = 'I thought too hard, sorry, try again.';
+            finalRes = finalRes.replace(/<\|separator\|>/g, '*no response*');
         }
+
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/local.js

@@ -1,4 +1,5 @@
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Local {
     constructor(model_name, url, params) {
@@ -10,11 +11,10 @@ export class Local {
     }
 
     async sendRequest(turns, systemMessage) {
-        let model = this.model_name || 'llama3.1'; // Updated to llama3.1, as it is more performant than llama3
+        let model = this.model_name || 'llama3.1';
         let messages = strictFormat(turns);
         messages.unshift({ role: 'system', content: systemMessage });
         
-        // We'll attempt up to 5 times for models with deepseek-r1-esk reasoning if the <think> tags are mismatched.
         const maxAttempts = 5;
         let attempt = 0;
         let finalRes = null;
@@ -24,14 +24,14 @@ export class Local {
             console.log(`Awaiting local response... (model: ${model}, attempt: ${attempt})`);
             let res = null;
             try {
-                res = await this.send(this.chat_endpoint, {
+                let apiResponse = await this.send(this.chat_endpoint, {
                     model: model,
                     messages: messages,
                     stream: false,
                     ...(this.params || {})
                 });
-                if (res) {
+                if (apiResponse) {
-                    res = res['message']['content'];
+                    res = apiResponse['message']['content'];
                 } else {
                     res = 'No response data.';
                 }
@@ -43,38 +43,33 @@ export class Local {
                     console.log(err);
                     res = 'My brain disconnected, try again.';
                 }
-
             }
 
-            // If the model name includes "deepseek-r1" or "Andy-3.5-reasoning", then handle the <think> block.
             const hasOpenTag = res.includes("<think>");
             const hasCloseTag = res.includes("</think>");
 
-                // If there's a partial mismatch, retry to get a complete response.
             if ((hasOpenTag && !hasCloseTag)) {
                 console.warn("Partial <think> block detected. Re-generating...");
-                    continue; 
+                if (attempt < maxAttempts) continue;
             }
-            
-                // If </think> is present but <think> is not, prepend <think>
             if (hasCloseTag && !hasOpenTag) {
                 res = '<think>' + res;
             }
-                // Changed this so if the model reasons, using <think> and </think> but doesn't start the message with <think>, <think> ges prepended to the message so no error occur.
-            
-                // If both tags appear, remove them (and everything inside).
             if (hasOpenTag && hasCloseTag) {
-                    res = res.replace(/<think>[\s\S]*?<\/think>/g, '');
+                res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
             }
-
             finalRes = res;
-            break; // Exit the loop if we got a valid response.
+            break;
         }
 
         if (finalRes == null) {
-            console.warn("Could not get a valid <think> block or normal response after max attempts.");
+            console.warn("Could not get a valid response after max attempts.");
             finalRes = 'I thought too hard, sorry, try again.';
         }
+        if (typeof finalRes === 'string') {
+            finalRes = finalRes.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), finalRes);
         return finalRes;
     }
 

src/models/mistral.js

@@ -1,59 +1,39 @@
 import { Mistral as MistralClient } from '@mistralai/mistralai';
 import { getKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Mistral {
     #client;
-
     constructor(model_name, url, params) {
         this.model_name = model_name;
         this.params = params;
 
         if (typeof url === "string") {
             console.warn("Mistral does not support custom URL's, ignoring!");
-
         }
-
         if (!getKey("MISTRAL_API_KEY")) {
             throw new Error("Mistral API Key missing, make sure to set MISTRAL_API_KEY in settings.json")
         }
+        this.#client = new MistralClient({ apiKey: getKey("MISTRAL_API_KEY") });
         
-        this.#client = new MistralClient(
+        if (typeof this.model_name === "string" && typeof this.model_name.split("/")[1] !== "undefined") {
-            {
+            this.model_name = this.model_name.split("/")[1];
-                apiKey: getKey("MISTRAL_API_KEY")
-            }
-        );
-
-        
-        // Prevents the following code from running when model not specified
-        if (typeof this.model_name === "undefined") return;
-
-        // get the model name without the "mistral" or "mistralai" prefix
-        // e.g "mistral/mistral-large-latest" -> "mistral-large-latest"
-        if (typeof model_name.split("/")[1] !== "undefined") {
-            this.model_name = model_name.split("/")[1];
         }
     }
 
     async sendRequest(turns, systemMessage) {
-
         let result;
-
-        try {
         const model = this.model_name || "mistral-large-latest";
-
+        const messages = [{ role: "system", content: systemMessage }];
-            const messages = [
-                { role: "system", content: systemMessage }
-            ];
         messages.push(...strictFormat(turns));
-
+        try {
             console.log('Awaiting mistral api response...')
             const response  = await this.#client.chat.complete({
                 model,
                 messages,
                 ...(this.params || {})
             });
-
             result = response.choices[0].message.content;
         } catch (err) {
             if (err.message.includes("A request containing images has been given to a model which does not have the 'vision' capability.")) {
@@ -63,24 +43,28 @@ export class Mistral {
             }
             console.log(err);
         }
-
+        if (typeof result === 'string') {
+            result = result.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), result);
         return result;
     }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+    async sendVisionRequest(original_turns, systemMessage, imageBuffer) {
-        const imageMessages = [...messages];
+        const imageFormattedTurns = [...original_turns];
-        imageMessages.push({
+        const userMessageContent = [{ type: "text", text: systemMessage }];
-            role: "user",
+        userMessageContent.push({
-            content: [
-                { type: "text", text: systemMessage },
-                {
             type: "image_url",
             imageUrl: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
-                }
-            ]
         });
+        imageFormattedTurns.push({ role: "user", content: userMessageContent });
         
-        return this.sendRequest(imageMessages, systemMessage);
+        const res = await this.sendRequest(imageFormattedTurns, systemMessage);
+
+        if (imageBuffer && res) {
+            logVision(original_turns, imageBuffer, res, systemMessage);
+        }
+        return res;
     }
 
     async embed(text) {

src/models/novita.js

@@ -1,6 +1,7 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 // llama, mistral
 export class Novita {
@@ -49,14 +50,23 @@ export class Novita {
               res = 'My brain disconnected, try again.';
           }
       }
-      if (res.includes('<think>')) {
+      if (typeof res === 'string') {
+          res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+      }
+      log(JSON.stringify(messages), res); // Log transformed res
+
+      // Existing stripping logic for <think> tags
+      if (res && typeof res === 'string' && res.includes('<think>')) {
           let start = res.indexOf('<think>');
-        let end = res.indexOf('</think>') + 8;
+          let end = res.indexOf('</think>') + 8; // length of '</think>'
-        if (start != -1) {
+          if (start !== -1) { // Ensure '<think>' was found
-          if (end != -1) {
+              if (end !== -1 && end > start + 7) { // Ensure '</think>' was found and is after '<think>'
                   res = res.substring(0, start) + res.substring(end);
               } else {
-            res = res.substring(0, start+7);
+                  // Malformed or missing end tag, strip from '<think>' onwards or handle as error
+                  // Original code: res = res.substring(0, start+7); This would leave "<think>"
+                  // Let's assume we strip from start if end is not valid.
+                  res = res.substring(0, start);
               }
           }
           res = res.trim();

src/models/openrouter.js

@@ -1,73 +1,105 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class OpenRouter {
     constructor(model_name, url) {
         this.model_name = model_name;
-
         let config = {};
         config.baseURL = url || 'https://openrouter.ai/api/v1';
-
         const apiKey = getKey('OPENROUTER_API_KEY');
         if (!apiKey) {
             console.error('Error: OPENROUTER_API_KEY not found. Make sure it is set properly.');
         }
-
-        // Pass the API key to OpenAI compatible Api
         config.apiKey = apiKey;
-
         this.openai = new OpenAIApi(config);
     }
 
-    async sendRequest(turns, systemMessage, stop_seq='*') {
+    async sendRequest(turns, systemMessage, stop_seq = '***', visionImageBuffer = null, visionMessage = null) {
-        let messages = [{ role: 'system', content: systemMessage }, ...turns];
+        let processedSystemMessage = systemMessage;
+
+        let messages = [{ role: 'system', content: processedSystemMessage }, ...turns];
         messages = strictFormat(messages);
 
-        // Choose a valid model from openrouter.ai (for example, "openai/gpt-4o")
         const pack = {
             model: this.model_name,
             messages,
-            stop: stop_seq
+            include_reasoning: true,
+            // stop: stop_seq // Commented out since some API providers on Openrouter do not support a stop sequence, such as Grok 3
         };
 
+        const maxAttempts = 5;
+        let attempt = 0;
+        let finalRes = null;
+
+        while (attempt < maxAttempts) {
+            attempt++;
+            console.info(`Awaiting openrouter API response... (attempt: ${attempt})`);
             let res = null;
             try {
-            console.log('Awaiting openrouter api response...');
                 let completion = await this.openai.chat.completions.create(pack);
                 if (!completion?.choices?.[0]) {
                     console.error('No completion or choices returned:', completion);
                     return 'No response received.';
                 }
+
+                const logMessages = [{ role: "system", content: processedSystemMessage }].concat(turns);
+
                 if (completion.choices[0].finish_reason === 'length') {
                     throw new Error('Context length exceeded');
                 }
-            console.log('Received.');
+                
+                if (completion.choices[0].message.reasoning) {
+                    try{
+                        const reasoning = '<think>\n' + completion.choices[0].message.reasoning + '</think>\n';
+                        const content = completion.choices[0].message.content;
+
+                        // --- VISION LOGGING ---
+                        if (visionImageBuffer) {
+                            logVision(turns, visionImageBuffer, reasoning + "\n" + content, visionMessage);
+                        } else {
+                            log(JSON.stringify(logMessages), reasoning + "\n" + content);
+                        }
+                        res = content;
+                    } catch {}
+                } else {
+                    try {
                         res = completion.choices[0].message.content;
+                        if (visionImageBuffer) {
+                            logVision(turns, visionImageBuffer, res, visionMessage);
+                        } else {
+                            log(JSON.stringify(logMessages), res);
+                        }
+                    } catch {
+                        console.warn("Unable to log due to unknown error!");
+                    }
+                }
+                // Trim <think> blocks from the final response if present.
+                if (res && res.includes("<think>") && res.includes("</think>")) {
+                    res = res.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+                }
+
+                console.info('Received.');
             } catch (err) {
                 console.error('Error while awaiting response:', err);
-            // If the error indicates a context-length problem, we can slice the turns array, etc.
                 res = 'My brain disconnected, try again.';
             }
-        return res;
+
+            finalRes = res;
+            break; // Exit loop once a valid response is obtained.
         }
 
-    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+        if (finalRes == null) {
-        const imageMessages = [...messages];
+            console.warn("Could not get a valid <think> block or normal response after max attempts.");
-        imageMessages.push({
+            finalRes = 'I thought too hard, sorry, try again.';
-            role: "user",
-            content: [
-                { type: "text", text: systemMessage },
-                {
-                    type: "image_url",
-                    image_url: {
-                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
         }
+        return finalRes;
     }
-            ]
-        });
 
-        return this.sendRequest(imageMessages, systemMessage);
+    // Vision request: pass visionImageBuffer and visionMessage
+    async sendVisionRequest(turns, systemMessage, imageBuffer, visionMessage = null, stop_seq = '***') {
+        return await this.sendRequest(turns, systemMessage, stop_seq, imageBuffer, visionMessage);
     }
 
     async embed(text) {

src/models/prompter.js

@@ -445,8 +445,26 @@ export class Prompter {
     }
 
     async _saveLog(prompt, messages, generation, tag) {
-        if (!settings.log_all_prompts)
+        // NEW LOGIC STARTS
+        switch (tag) {
+            case 'conversation':
+            case 'coding': // Assuming coding logs fall under normal data
+            case 'memSaving':
+                if (!settings.log_normal_data) return;
+                break;
+            // Add case for 'vision' if prompter.js starts logging vision prompts/responses via _saveLog
+            // case 'vision':
+            //     if (!settings.log_vision_data) return;
+            //     break;
+            default:
+                // If it's an unknown tag, perhaps log it if general logging is on, or ignore.
+                // For safety, let's assume if it's not specified, it doesn't get logged unless a general flag is on.
+                // However, the goal is to use specific flags. So, if a new tag appears, this logic should be updated.
+                // For now, if it doesn't match known tags that map to a setting, it won't log.
                 return;
+        }
+        // NEW LOGIC ENDS
+
         const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
         let logEntry;
         let task_id = this.agent.task.task_id;

src/models/qwen.js

@@ -1,6 +1,7 @@
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class Qwen {
     constructor(model_name, url, params) {
@@ -45,6 +46,10 @@ export class Qwen {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }
 

src/models/replicate.js

@@ -1,6 +1,7 @@
 import Replicate from 'replicate';
 import { toSinglePrompt } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
+import { log, logVision } from '../../logger.js';
 
 // llama, mistral
 export class ReplicateAPI {
@@ -23,6 +24,7 @@ export class ReplicateAPI {
 		const prompt = toSinglePrompt(turns, null, stop_seq);
 		let model_name = this.model_name || 'meta/meta-llama-3-70b-instruct';
 
+		const logInputMessages = [{role: 'system', content: systemMessage}, ...turns];
 		const input = { 
 			prompt, 
 			system_prompt: systemMessage,
@@ -45,6 +47,10 @@ export class ReplicateAPI {
 			console.log(err);
 			res = 'My brain disconnected, try again.';
 		}
+		if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+		log(JSON.stringify(logInputMessages), res);
 		console.log('Received.');
 		return res;
 	}

src/models/vllm.js

@@ -1,9 +1,13 @@
 // This code uses Dashscope and HTTP to ensure the latest support for the Qwen model.
 // Qwen is also compatible with the OpenAI API format;
 
+// This code uses Dashscope and HTTP to ensure the latest support for the Qwen model.
+// Qwen is also compatible with the OpenAI API format;
+
 import OpenAIApi from 'openai';
 import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
+import { log, logVision } from '../../logger.js';
 
 export class VLLM {
     constructor(model_name, url) {
@@ -53,6 +57,10 @@ export class VLLM {
                 res = 'My brain disconnected, try again.';
             }
         }
+        if (typeof res === 'string') {
+            res = res.replace(/<thinking>/g, '<think>').replace(/<\/thinking>/g, '</think>');
+        }
+        log(JSON.stringify(messages), res);
         return res;
     }