Merge branch 'danny-avila:main' into main

.env.example

@@ -76,13 +76,14 @@ PROXY=
 # SHUTTLEAI_API_KEY=
 # TOGETHERAI_API_KEY=
 # UNIFY_API_KEY=
+# XAI_API_KEY=
 
 #============#
 # Anthropic  #
 #============#
 
 ANTHROPIC_API_KEY=user_provided
-# ANTHROPIC_MODELS=claude-3-5-sonnet-20241022,claude-3-5-sonnet-latest,claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
+# ANTHROPIC_MODELS=claude-3-5-haiku-20241022,claude-3-5-sonnet-20241022,claude-3-5-sonnet-latest,claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
 # ANTHROPIC_REVERSE_PROXY=
 
 #============#
@@ -302,6 +303,7 @@ TTS_API_KEY=
 
 # RAG_OPENAI_BASEURL=
 # RAG_OPENAI_API_KEY=
+# RAG_USE_FULL_CONTEXT=
 # EMBEDDINGS_PROVIDER=openai
 # EMBEDDINGS_MODEL=text-embedding-3-small
 

api/app/clients/AnthropicClient.js

@@ -98,8 +98,8 @@ class AnthropicClient extends BaseClient {
     );
 
     const modelMatch = matchModelName(this.modelOptions.model, EModelEndpoint.anthropic);
-    this.isClaude3 = modelMatch.startsWith('claude-3');
-    this.isLegacyOutput = !modelMatch.startsWith('claude-3-5-sonnet');
+    this.isClaude3 = modelMatch.includes('claude-3');
+    this.isLegacyOutput = !modelMatch.includes('claude-3-5-sonnet');
     this.supportsCacheControl =
       this.options.promptCache && this.checkPromptCacheSupport(modelMatch);
 
@@ -634,7 +634,7 @@ class AnthropicClient extends BaseClient {
       );
     };
 
-    if (this.modelOptions.model.startsWith('claude-3')) {
+    if (this.modelOptions.model.includes('claude-3')) {
       await buildMessagesPayload();
       processTokens();
       return {
@@ -687,6 +687,7 @@ class AnthropicClient extends BaseClient {
     }
     if (
       modelMatch === 'claude-3-5-sonnet' ||
+      modelMatch === 'claude-3-5-haiku' ||
       modelMatch === 'claude-3-haiku' ||
       modelMatch === 'claude-3-opus'
     ) {

api/app/clients/ChatGPTClient.js

@@ -678,7 +678,6 @@ ${botMessage.message}
 
     const instructionsPayload = {
       role: 'system',
-      name: 'instructions',
       content: promptPrefix,
     };
 

api/app/clients/OpenAIClient.js

@@ -553,7 +553,6 @@ class OpenAIClient extends BaseClient {
       promptPrefix = `Instructions:\n${promptPrefix.trim()}`;
       instructions = {
         role: 'system',
-        name: 'instructions',
         content: promptPrefix,
       };
 

api/app/clients/PluginsClient.js

@@ -458,7 +458,6 @@ class PluginsClient extends OpenAIClient {
 
     const instructionsPayload = {
       role: 'system',
-      name: 'instructions',
       content: promptPrefix,
     };
 

api/app/clients/prompts/formatMessages.js

@@ -217,9 +217,41 @@ const formatAgentMessages = (payload) => {
   return messages;
 };
 
+/**
+ * Formats an array of messages for LangChain, making sure all content fields are strings
+ * @param {Array<(HumanMessage|AIMessage|SystemMessage|ToolMessage)>} payload - The array of messages to format.
+ * @returns {Array<(HumanMessage|AIMessage|SystemMessage|ToolMessage)>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ */
+const formatContentStrings = (payload) => {
+  const messages = [];
+
+  for (const message of payload) {
+    if (typeof message.content === 'string') {
+      continue;
+    }
+
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+
+    // Reduce text types to a single string, ignore all other types
+    const content = message.content.reduce((acc, curr) => {
+      if (curr.type === ContentTypes.TEXT) {
+        return `${acc}${curr[ContentTypes.TEXT]}\n`;
+      }
+      return acc;
+    }, '');
+
+    message.content = content.trim();
+  }
+
+  return messages;
+};
+
 module.exports = {
   formatMessage,
   formatFromLangChain,
   formatAgentMessages,
+  formatContentStrings,
   formatLangChainMessages,
 };

api/app/clients/specs/OpenAIClient.test.js

@@ -446,7 +446,7 @@ describe('OpenAIClient', () => {
         promptPrefix: 'Test Prefix',
       });
       expect(result).toHaveProperty('prompt');
-      const instructions = result.prompt.find((item) => item.name === 'instructions');
+      const instructions = result.prompt.find((item) => item.content.includes('Test Prefix'));
       expect(instructions).toBeDefined();
       expect(instructions.content).toContain('Test Prefix');
     });
@@ -476,15 +476,17 @@ describe('OpenAIClient', () => {
       const result = await client.buildMessages(messages, parentMessageId, {
         isChatCompletion: true,
       });
-      const instructions = result.prompt.find((item) => item.name === 'instructions');
+      const instructions = result.prompt.find((item) =>
+        item.content.includes('Test Prefix from options'),
+      );
       expect(instructions.content).toContain('Test Prefix from options');
     });
 
     it('should handle case when neither promptPrefix argument nor options.promptPrefix is set', async () => {
       const result = await client.buildMessages(messages, parentMessageId, {
         isChatCompletion: true,
       });
-      const instructions = result.prompt.find((item) => item.name === 'instructions');
+      const instructions = result.prompt.find((item) => item.content.includes('Test Prefix'));
       expect(instructions).toBeUndefined();
     });
 

api/app/clients/tools/util/handleTools.js

@@ -24,6 +24,7 @@ const {
   StructuredWolfram,
   TavilySearchResults,
 } = require('../');
+const { primeFiles } = require('~/server/services/Files/Code/process');
 const createFileSearchTool = require('./createFileSearchTool');
 const { loadToolSuite } = require('./loadToolSuite');
 const { loadSpecs } = require('./loadSpecs');
@@ -255,12 +256,14 @@ const loadTools = async ({
   for (const tool of tools) {
     if (tool === Tools.execute_code) {
       const authValues = await loadAuthValues({
-        userId: user.id,
+        userId: user,
         authFields: [EnvVar.CODE_API_KEY],
       });
+      const files = await primeFiles(options, authValues[EnvVar.CODE_API_KEY]);
       requestedTools[tool] = () =>
         createCodeExecutionTool({
-          user_id: user.id,
+          user_id: user,
+          files,
           ...authValues,
         });
       continue;

api/models/schema/fileSchema.js

@@ -21,6 +21,8 @@ const mongoose = require('mongoose');
  * @property {string} [source] - The source of the file (e.g., from FileSources)
  * @property {number} [width] - Optional width of the file
  * @property {number} [height] - Optional height of the file
+ * @property {Object} [metadata] - Metadata related to the file
+ * @property {string} [metadata.fileIdentifier] - Unique identifier for the file in metadata
  * @property {Date} [expiresAt] - Optional expiration date of the file
  * @property {Date} [createdAt] - Date when the file was created
  * @property {Date} [updatedAt] - Date when the file was updated
@@ -91,6 +93,9 @@ const fileSchema = mongoose.Schema(
     },
     width: Number,
     height: Number,
+    metadata: {
+      fileIdentifier: String,
+    },
     expiresAt: {
       type: Date,
       expires: 3600, // 1 hour in seconds

api/models/tx.js

@@ -10,6 +10,13 @@ const bedrockValues = {
   'llama3-1-8b': { prompt: 0.3, completion: 0.6 },
   'llama3-1-70b': { prompt: 2.65, completion: 3.5 },
   'llama3-1-405b': { prompt: 5.32, completion: 16.0 },
+  'llama2:13b': { prompt: 0.75, completion: 1.0 },
+  'llama2:70b': { prompt: 1.95, completion: 2.56 },
+  'llama3:8b': { prompt: 0.3, completion: 0.6 },
+  'llama3:70b': { prompt: 2.65, completion: 3.5 },
+  'llama3.1:8b': { prompt: 0.3, completion: 0.6 },
+  'llama3.1:70b': { prompt: 2.65, completion: 3.5 },
+  'llama3.1:405b': { prompt: 5.32, completion: 16.0 },
   'mistral-7b': { prompt: 0.15, completion: 0.2 },
   'mistral-small': { prompt: 0.15, completion: 0.2 },
   'mixtral-8x7b': { prompt: 0.45, completion: 0.7 },
@@ -49,6 +56,8 @@ const tokenValues = Object.assign(
     'claude-3-sonnet': { prompt: 3, completion: 15 },
     'claude-3-5-sonnet': { prompt: 3, completion: 15 },
     'claude-3.5-sonnet': { prompt: 3, completion: 15 },
+    'claude-3-5-haiku': { prompt: 1, completion: 5 },
+    'claude-3.5-haiku': { prompt: 1, completion: 5 },
     'claude-3-haiku': { prompt: 0.25, completion: 1.25 },
     'claude-2.1': { prompt: 8, completion: 24 },
     'claude-2': { prompt: 8, completion: 24 },
@@ -74,6 +83,8 @@ const tokenValues = Object.assign(
 const cacheTokenValues = {
   'claude-3.5-sonnet': { write: 3.75, read: 0.3 },
   'claude-3-5-sonnet': { write: 3.75, read: 0.3 },
+  'claude-3.5-haiku': { write: 1.25, read: 0.1 },
+  'claude-3-5-haiku': { write: 1.25, read: 0.1 },
   'claude-3-haiku': { write: 0.3, read: 0.03 },
 };
 

api/models/tx.spec.js

@@ -92,6 +92,20 @@ describe('getValueKey', () => {
     expect(getValueKey('claude-3.5-sonnet-turbo')).toBe('claude-3.5-sonnet');
     expect(getValueKey('claude-3.5-sonnet-0125')).toBe('claude-3.5-sonnet');
   });
+
+  it('should return "claude-3-5-haiku" for model type of "claude-3-5-haiku-"', () => {
+    expect(getValueKey('claude-3-5-haiku-20240620')).toBe('claude-3-5-haiku');
+    expect(getValueKey('anthropic/claude-3-5-haiku')).toBe('claude-3-5-haiku');
+    expect(getValueKey('claude-3-5-haiku-turbo')).toBe('claude-3-5-haiku');
+    expect(getValueKey('claude-3-5-haiku-0125')).toBe('claude-3-5-haiku');
+  });
+
+  it('should return "claude-3.5-haiku" for model type of "claude-3.5-haiku-"', () => {
+    expect(getValueKey('claude-3.5-haiku-20240620')).toBe('claude-3.5-haiku');
+    expect(getValueKey('anthropic/claude-3.5-haiku')).toBe('claude-3.5-haiku');
+    expect(getValueKey('claude-3.5-haiku-turbo')).toBe('claude-3.5-haiku');
+    expect(getValueKey('claude-3.5-haiku-0125')).toBe('claude-3.5-haiku');
+  });
 });
 
 describe('getMultiplier', () => {
@@ -248,6 +262,8 @@ describe('getCacheMultiplier', () => {
   it('should return the correct cache multiplier for a given valueKey and cacheType', () => {
     expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'write' })).toBe(3.75);
     expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'read' })).toBe(0.3);
+    expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'write' })).toBe(1.25);
+    expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'read' })).toBe(0.1);
     expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'write' })).toBe(0.3);
     expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'read' })).toBe(0.03);
   });

api/package.json

@@ -43,7 +43,7 @@
     "@langchain/core": "^0.2.18",
     "@langchain/google-genai": "^0.0.11",
     "@langchain/google-vertexai": "^0.0.17",
-    "@librechat/agents": "^1.6.9",
+    "@librechat/agents": "^1.7.7",
     "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
     "cheerio": "^1.0.0-rc.12",

api/server/controllers/PluginController.js

@@ -1,5 +1,5 @@
 const { promises: fs } = require('fs');
-const { CacheKeys } = require('librechat-data-provider');
+const { CacheKeys, AuthType } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
 const { getLogStores } = require('~/cache');
 
@@ -25,7 +25,7 @@ const filterUniquePlugins = (plugins) => {
  * @param {TPlugin} plugin The plugin object containing the authentication configuration.
  * @returns {boolean} True if the plugin is authenticated for all required fields, false otherwise.
  */
-const isPluginAuthenticated = (plugin) => {
+const checkPluginAuth = (plugin) => {
   if (!plugin.authConfig || plugin.authConfig.length === 0) {
     return false;
   }
@@ -36,7 +36,7 @@ const isPluginAuthenticated = (plugin) => {
 
     for (const fieldOption of authFieldOptions) {
       const envValue = process.env[fieldOption];
-      if (envValue && envValue.trim() !== '' && envValue !== 'user_provided') {
+      if (envValue && envValue.trim() !== '' && envValue !== AuthType.USER_PROVIDED) {
         isFieldAuthenticated = true;
         break;
       }
@@ -64,7 +64,7 @@ const getAvailablePluginsController = async (req, res) => {
     let authenticatedPlugins = [];
     for (const plugin of uniquePlugins) {
       authenticatedPlugins.push(
-        isPluginAuthenticated(plugin) ? { ...plugin, authenticated: true } : plugin,
+        checkPluginAuth(plugin) ? { ...plugin, authenticated: true } : plugin,
       );
     }
 
@@ -111,7 +111,7 @@ const getAvailableTools = async (req, res) => {
     const uniquePlugins = filterUniquePlugins(jsonData);
 
     const authenticatedPlugins = uniquePlugins.map((plugin) => {
-      if (isPluginAuthenticated(plugin)) {
+      if (checkPluginAuth(plugin)) {
         return { ...plugin, authenticated: true };
       } else {
         return plugin;

api/server/controllers/UserController.js

@@ -61,10 +61,10 @@ const deleteUserFiles = async (req) => {
 
 const updateUserPluginsController = async (req, res) => {
   const { user } = req;
-  const { pluginKey, action, auth, isAssistantTool } = req.body;
+  const { pluginKey, action, auth, isEntityTool } = req.body;
   let authService;
   try {
-    if (!isAssistantTool) {
+    if (!isEntityTool) {
       const userPluginsService = await updateUserPluginsService(user, pluginKey, action);
 
       if (userPluginsService instanceof Error) {

api/server/controllers/agents/callbacks.js

@@ -1,6 +1,12 @@
 const { Tools } = require('librechat-data-provider');
-const { GraphEvents, ToolEndHandler, ChatModelStreamHandler } = require('@librechat/agents');
+const {
+  EnvVar,
+  GraphEvents,
+  ToolEndHandler,
+  ChatModelStreamHandler,
+} = require('@librechat/agents');
 const { processCodeOutput } = require('~/server/services/Files/Code/process');
+const { loadAuthValues } = require('~/app/clients/tools/util');
 const { logger } = require('~/config');
 
 /** @typedef {import('@librechat/agents').Graph} Graph */
@@ -158,13 +164,18 @@ function createToolEndCallback({ req, res, artifactPromises }) {
       const { id, name } = file;
       artifactPromises.push(
         (async () => {
+          const result = await loadAuthValues({
+            userId: req.user.id,
+            authFields: [EnvVar.CODE_API_KEY],
+          });
           const fileMetadata = await processCodeOutput({
             req,
             id,
             name,
+            apiKey: result[EnvVar.CODE_API_KEY],
             toolCallId: tool_call_id,
             messageId: metadata.run_id,
-            sessionId: artifact.session_id,
+            session_id: artifact.session_id,
             conversationId: metadata.thread_id,
           });
           if (!res.headersSent) {