From c6a765a8b39f540ae3c227ee26fde315d901e00a Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Fri, 31 Jan 2025 16:37:11 +1100 Subject: [PATCH 1/6] wip --- crates/goose/src/providers/openrouter.rs | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index 1750776b8..67ee1ebb0 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -14,6 +14,7 @@ use mcp_core::tool::Tool; pub const OPENROUTER_DEFAULT_MODEL: &str = "anthropic/claude-3.5-sonnet"; pub const OPENROUTER_MODEL_PREFIX_ANTHROPIC: &str = "anthropic"; +pub const OPENROUTER_MODEL_PREFIX_DEEPSEEK: &str = "deepseek-r1"; // OpenRouter can run many models, we suggest the default pub const OPENROUTER_KNOWN_MODELS: &[&str] = &[OPENROUTER_DEFAULT_MODEL]; @@ -132,6 +133,63 @@ fn update_request_for_anthropic(original_payload: &Value) -> Value { payload } +fn update_request_for_deepseek(original_payload: &Value) -> Value { + let mut payload = original_payload.clone(); + + if let Some(messages_spec) = payload + .as_object_mut() + .and_then(|obj| obj.get_mut("messages")) + .and_then(|messages| messages.as_array_mut()) + { + // Add "cache_control" to the last and second-to-last "user" messages + let mut user_count = 0; + for message in messages_spec.iter_mut().rev() { + if message.get("role") == Some(&json!("user")) { + if let Some(content) = message.get_mut("content") { + if let Some(content_str) = content.as_str() { + *content = json!([{ + "type": "text", + "text": content_str, + "cache_control": { "type": "ephemeral" } + }]); + } + } + user_count += 1; + if user_count >= 2 { + break; + } + } + } + + // Update the system message to have cache_control field + if let Some(system_message) = messages_spec + .iter_mut() + .find(|msg| msg.get("role") == Some(&json!("system"))) + { + if let Some(content) = system_message.get_mut("content") { + if let Some(content_str) = content.as_str() { + *system_message = json!({ + "role": "system", + "content": [{ + "type": "text", + "text": content_str, + "cache_control": { "type": "ephemeral" } + }] + }); + } + } + } + } + + // Remove any tools/function calling capabilities + if let Some(obj) = payload.as_object_mut() { + obj.remove("tools"); + obj.remove("tool_choice"); + } + + payload +} + fn create_request_based_on_model( model_config: &ModelConfig, system: &str, @@ -146,12 +204,21 @@ fn create_request_based_on_model( &super::utils::ImageFormat::OpenAi, )?; + // Check for Anthropic models if model_config .model_name .starts_with(OPENROUTER_MODEL_PREFIX_ANTHROPIC) { payload = update_request_for_anthropic(&payload); } + + // Check for DeepSeek models + if model_config + .model_name + .contains(OPENROUTER_MODEL_PREFIX_DEEPSEEK) + { + payload = update_request_for_deepseek(&payload); + } Ok(payload) } From 25d21359f4149e9b4ca9eaf4428b68d1799d73af Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Fri, 31 Jan 2025 16:56:05 +1100 Subject: [PATCH 2/6] debugging --- crates/goose/src/providers/openrouter.rs | 54 +++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index 67ee1ebb0..3244de8e1 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -75,6 +75,23 @@ impl OpenRouterProvider { handle_response_openai_compat(response).await } + + // Extract tools from system message if it contains a tools section + fn extract_tools_from_system(system: &str) -> Option> { + if let Some(tools_start) = system.find("") { + if let Some(tools_end) = system.find("") { + let tools_text = &system[tools_start..=tools_end + 11]; // +11 to include "" + match serde_json::from_str::>(tools_text) { + Ok(tools) => return Some(tools), + Err(e) => { + tracing::warn!("Failed to parse tools from system message: {}", e); + return None; + } + } + } + } + None + } } /// Update the request when using anthropic model. @@ -136,6 +153,9 @@ fn update_request_for_anthropic(original_payload: &Value) -> Value { fn update_request_for_deepseek(original_payload: &Value) -> Value { let mut payload = original_payload.clone(); + // Extract tools before removing them from the payload + let tools = payload.get("tools").cloned(); + if let Some(messages_spec) = payload .as_object_mut() .and_then(|obj| obj.get_mut("messages")) @@ -161,18 +181,31 @@ fn update_request_for_deepseek(original_payload: &Value) -> Value { } } - // Update the system message to have cache_control field + // Update the system message to include tools and have cache_control field if let Some(system_message) = messages_spec .iter_mut() .find(|msg| msg.get("role") == Some(&json!("system"))) { if let Some(content) = system_message.get_mut("content") { if let Some(content_str) = content.as_str() { + let system_content = if let Some(tools) = tools { + // Format tools as a string + let tools_str = serde_json::to_string_pretty(&tools) + .unwrap_or_else(|_| "[]".to_string()); + format!( + "{}\n\nYou have access to the following tools:\n\n{}\n\n\nTo use a tool, respond with a message containing an XML-like function call block like this:\n\n\nvalue\n\n", + content_str, + tools_str + ) + } else { + content_str.to_string() + }; + *system_message = json!({ "role": "system", "content": [{ "type": "text", - "text": content_str, + "text": system_content, "cache_control": { "type": "ephemeral" } }] }); @@ -181,7 +214,7 @@ fn update_request_for_deepseek(original_payload: &Value) -> Value { } } - // Remove any tools/function calling capabilities + // Remove any tools/function calling capabilities from the request if let Some(obj) = payload.as_object_mut() { obj.remove("tools"); obj.remove("tool_choice"); @@ -262,14 +295,25 @@ impl Provider for OpenRouterProvider { messages: &[Message], tools: &[Tool], ) -> Result<(Message, ProviderUsage), ProviderError> { + // Try to extract tools from system message if available + let effective_tools = if let Some(system_tools) = Self::extract_tools_from_system(system) { + system_tools + } else { + tools.to_vec() + }; + // Create the base payload - let payload = create_request_based_on_model(&self.model, system, messages, tools)?; + let payload = create_request_based_on_model(&self.model, system, messages, &effective_tools)?; // Make request let response = self.post(payload.clone()).await?; // Parse response let message = response_to_message(response.clone())?; + + // Debug log the response structure + tracing::debug!("OpenRouter response: {}", serde_json::to_string_pretty(&response).unwrap_or_default()); + let usage = match get_usage(&response) { Ok(usage) => usage, Err(ProviderError::UsageError(e)) => { @@ -282,4 +326,4 @@ impl Provider for OpenRouterProvider { emit_debug_trace(self, &payload, &response, &usage); Ok((message, ProviderUsage::new(model, usage))) } -} +} \ No newline at end of file From c64d5a5094cbd2f57a6df2c6c2e72f682de641cc Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Fri, 31 Jan 2025 17:00:24 +1100 Subject: [PATCH 3/6] progress --- crates/goose/src/providers/openrouter.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index 3244de8e1..168b4712b 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -244,7 +244,7 @@ fn create_request_based_on_model( { payload = update_request_for_anthropic(&payload); } - + // Check for DeepSeek models if model_config .model_name @@ -303,17 +303,21 @@ impl Provider for OpenRouterProvider { }; // Create the base payload - let payload = create_request_based_on_model(&self.model, system, messages, &effective_tools)?; + let payload = + create_request_based_on_model(&self.model, system, messages, &effective_tools)?; // Make request let response = self.post(payload.clone()).await?; // Parse response let message = response_to_message(response.clone())?; - + // Debug log the response structure - tracing::debug!("OpenRouter response: {}", serde_json::to_string_pretty(&response).unwrap_or_default()); - + println!( + "OpenRouter response: {}", + serde_json::to_string_pretty(&response).unwrap_or_default() + ); + let usage = match get_usage(&response) { Ok(usage) => usage, Err(ProviderError::UsageError(e)) => { @@ -326,4 +330,4 @@ impl Provider for OpenRouterProvider { emit_debug_trace(self, &payload, &response, &usage); Ok((message, ProviderUsage::new(model, usage))) } -} \ No newline at end of file +} From 6d1cda67dda1c53fbbabe75a91b1b72c3d9f4cf9 Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Sat, 1 Feb 2025 08:45:47 +1100 Subject: [PATCH 4/6] tidy up --- crates/goose/src/providers/openrouter.rs | 203 +++++++++++++++++------ 1 file changed, 149 insertions(+), 54 deletions(-) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index 168b4712b..a1a63f15e 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -7,10 +7,10 @@ use std::time::Duration; use super::base::{ConfigKey, Provider, ProviderMetadata, ProviderUsage, Usage}; use super::errors::ProviderError; use super::utils::{emit_debug_trace, get_model, handle_response_openai_compat}; -use crate::message::Message; +use crate::message::{Message, MessageContent}; use crate::model::ModelConfig; use crate::providers::formats::openai::{create_request, get_usage, response_to_message}; -use mcp_core::tool::Tool; +use mcp_core::tool::{Tool, ToolCall}; pub const OPENROUTER_DEFAULT_MODEL: &str = "anthropic/claude-3.5-sonnet"; pub const OPENROUTER_MODEL_PREFIX_ANTHROPIC: &str = "anthropic"; @@ -95,54 +95,61 @@ impl OpenRouterProvider { } /// Update the request when using anthropic model. -/// For anthropic model, we can enable prompt caching to save cost. Since openrouter is the OpenAI compatible -/// endpoint, we need to modify the open ai request to have anthropic cache control field. +/// For older anthropic models we enabled prompt caching, but newer ones (Claude-3) don't support it. fn update_request_for_anthropic(original_payload: &Value) -> Value { let mut payload = original_payload.clone(); - if let Some(messages_spec) = payload - .as_object_mut() - .and_then(|obj| obj.get_mut("messages")) - .and_then(|messages| messages.as_array_mut()) + // Only add cache control for non-Claude-3 models + if !payload + .get("model") + .and_then(|m| m.as_str()) + .unwrap_or("") + .contains("claude-3") { - // Add "cache_control" to the last and second-to-last "user" messages. - // During each turn, we mark the final message with cache_control so the conversation can be - // incrementally cached. The second-to-last user message is also marked for caching with the - // cache_control parameter, so that this checkpoint can read from the previous cache. - let mut user_count = 0; - for message in messages_spec.iter_mut().rev() { - if message.get("role") == Some(&json!("user")) { - if let Some(content) = message.get_mut("content") { - if let Some(content_str) = content.as_str() { - *content = json!([{ - "type": "text", - "text": content_str, - "cache_control": { "type": "ephemeral" } - }]); + if let Some(messages_spec) = payload + .as_object_mut() + .and_then(|obj| obj.get_mut("messages")) + .and_then(|messages| messages.as_array_mut()) + { + // Add "cache_control" to the last and second-to-last "user" messages. + // During each turn, we mark the final message with cache_control so the conversation can be + // incrementally cached. The second-to-last user message is also marked for caching with the + // cache_control parameter, so that this checkpoint can read from the previous cache. + let mut user_count = 0; + for message in messages_spec.iter_mut().rev() { + if message.get("role") == Some(&json!("user")) { + if let Some(content) = message.get_mut("content") { + if let Some(content_str) = content.as_str() { + *content = json!([{ + "type": "text", + "text": content_str, + "cache_control": { "type": "ephemeral" } + }]); + } + } + user_count += 1; + if user_count >= 2 { + break; } - } - user_count += 1; - if user_count >= 2 { - break; } } - } - // Update the system message to have cache_control field. - if let Some(system_message) = messages_spec - .iter_mut() - .find(|msg| msg.get("role") == Some(&json!("system"))) - { - if let Some(content) = system_message.get_mut("content") { - if let Some(content_str) = content.as_str() { - *system_message = json!({ - "role": "system", - "content": [{ - "type": "text", - "text": content_str, - "cache_control": { "type": "ephemeral" } - }] - }); + // Update the system message to have cache_control field. + if let Some(system_message) = messages_spec + .iter_mut() + .find(|msg| msg.get("role") == Some(&json!("system"))) + { + if let Some(content) = system_message.get_mut("content") { + if let Some(content_str) = content.as_str() { + *system_message = json!({ + "role": "system", + "content": [{ + "type": "text", + "text": content_str, + "cache_control": { "type": "ephemeral" } + }] + }); + } } } } @@ -295,29 +302,117 @@ impl Provider for OpenRouterProvider { messages: &[Message], tools: &[Tool], ) -> Result<(Message, ProviderUsage), ProviderError> { - // Try to extract tools from system message if available - let effective_tools = if let Some(system_tools) = Self::extract_tools_from_system(system) { - system_tools - } else { - tools.to_vec() - }; - // Create the base payload - let payload = - create_request_based_on_model(&self.model, system, messages, &effective_tools)?; + let payload = create_request_based_on_model(&self.model, system, messages, tools)?; // Make request let response = self.post(payload.clone()).await?; - // Parse response - let message = response_to_message(response.clone())?; - // Debug log the response structure println!( "OpenRouter response: {}", serde_json::to_string_pretty(&response).unwrap_or_default() ); + // First try to parse as OpenAI format + let mut message = response_to_message(response.clone())?; + + // If no tool calls were found in OpenAI format, check for XML format + if !message.is_tool_call() { + if let Some(MessageContent::Text(text_content)) = message.content.first() { + let content = &text_content.text; + if let Some(calls_start) = content.find("") { + if let Some(calls_end) = content.find("") { + let calls_text = &content[calls_start..=calls_end + 15]; + + // Extract the invoke block + if let Some(invoke_start) = calls_text.find("") { + let invoke_text = + &calls_text[invoke_start..invoke_start + invoke_end + 9]; + + // Parse name and parameters + if let Some(name_start) = invoke_text.find("name=\"") { + if let Some(name_end) = invoke_text[name_start + 6..].find("\"") + { + let name = invoke_text + [name_start + 6..name_start + 6 + name_end] + .to_string(); + + // Build parameters map + let mut parameters = serde_json::Map::new(); + let mut param_pos = 0; + while let Some(param_start) = + invoke_text[param_pos..].find("") + { + let param_text = &invoke_text[param_pos + + param_start + ..param_pos + param_start + param_end + 11]; + + if let Some(param_name_start) = + param_text.find("name=\"") + { + if let Some(param_name_end) = param_text + [param_name_start + 6..] + .find("\"") + { + let param_name = ¶m_text + [param_name_start + 6 + ..param_name_start + + 6 + + param_name_end]; + + if let Some(value_start) = + param_text.find(">") + { + if let Some(value_end) = param_text + [value_start + 1..] + .find("<") + { + let param_value = ¶m_text + [value_start + 1 + ..value_start + + 1 + + value_end]; + parameters.insert( + param_name.to_string(), + Value::String( + param_value.to_string(), + ), + ); + } + } + } + } + param_pos += param_start + param_end + 11; + } else { + break; + } + } + + // Create tool request + message.content.clear(); + message.content.push(MessageContent::tool_request( + "1", + Ok(ToolCall { + name, + arguments: serde_json::to_value(parameters) + .unwrap_or_default(), + }), + )); + } + } + } + } + } + } + } + } + let usage = match get_usage(&response) { Ok(usage) => usage, Err(ProviderError::UsageError(e)) => { From 29e8dc31ff8725199ba47b281d4353dedb3e2384 Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Sat, 1 Feb 2025 08:46:29 +1100 Subject: [PATCH 5/6] tidy up --- crates/goose/src/providers/openrouter.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index a1a63f15e..757515037 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -75,23 +75,6 @@ impl OpenRouterProvider { handle_response_openai_compat(response).await } - - // Extract tools from system message if it contains a tools section - fn extract_tools_from_system(system: &str) -> Option> { - if let Some(tools_start) = system.find("") { - if let Some(tools_end) = system.find("") { - let tools_text = &system[tools_start..=tools_end + 11]; // +11 to include "" - match serde_json::from_str::>(tools_text) { - Ok(tools) => return Some(tools), - Err(e) => { - tracing::warn!("Failed to parse tools from system message: {}", e); - return None; - } - } - } - } - None - } } /// Update the request when using anthropic model. From 2670533ca0019e343310e366e1dfec946773a56a Mon Sep 17 00:00:00 2001 From: Mic Neale Date: Sat, 1 Feb 2025 09:31:06 +1100 Subject: [PATCH 6/6] remove cache control --- crates/goose/src/providers/openrouter.rs | 98 +----------------------- 1 file changed, 2 insertions(+), 96 deletions(-) diff --git a/crates/goose/src/providers/openrouter.rs b/crates/goose/src/providers/openrouter.rs index 757515037..35a66478e 100644 --- a/crates/goose/src/providers/openrouter.rs +++ b/crates/goose/src/providers/openrouter.rs @@ -93,49 +93,7 @@ fn update_request_for_anthropic(original_payload: &Value) -> Value { .as_object_mut() .and_then(|obj| obj.get_mut("messages")) .and_then(|messages| messages.as_array_mut()) - { - // Add "cache_control" to the last and second-to-last "user" messages. - // During each turn, we mark the final message with cache_control so the conversation can be - // incrementally cached. The second-to-last user message is also marked for caching with the - // cache_control parameter, so that this checkpoint can read from the previous cache. - let mut user_count = 0; - for message in messages_spec.iter_mut().rev() { - if message.get("role") == Some(&json!("user")) { - if let Some(content) = message.get_mut("content") { - if let Some(content_str) = content.as_str() { - *content = json!([{ - "type": "text", - "text": content_str, - "cache_control": { "type": "ephemeral" } - }]); - } - } - user_count += 1; - if user_count >= 2 { - break; - } - } - } - - // Update the system message to have cache_control field. - if let Some(system_message) = messages_spec - .iter_mut() - .find(|msg| msg.get("role") == Some(&json!("system"))) - { - if let Some(content) = system_message.get_mut("content") { - if let Some(content_str) = content.as_str() { - *system_message = json!({ - "role": "system", - "content": [{ - "type": "text", - "text": content_str, - "cache_control": { "type": "ephemeral" } - }] - }); - } - } - } - } + {} } payload } @@ -150,59 +108,7 @@ fn update_request_for_deepseek(original_payload: &Value) -> Value { .as_object_mut() .and_then(|obj| obj.get_mut("messages")) .and_then(|messages| messages.as_array_mut()) - { - // Add "cache_control" to the last and second-to-last "user" messages - let mut user_count = 0; - for message in messages_spec.iter_mut().rev() { - if message.get("role") == Some(&json!("user")) { - if let Some(content) = message.get_mut("content") { - if let Some(content_str) = content.as_str() { - *content = json!([{ - "type": "text", - "text": content_str, - "cache_control": { "type": "ephemeral" } - }]); - } - } - user_count += 1; - if user_count >= 2 { - break; - } - } - } - - // Update the system message to include tools and have cache_control field - if let Some(system_message) = messages_spec - .iter_mut() - .find(|msg| msg.get("role") == Some(&json!("system"))) - { - if let Some(content) = system_message.get_mut("content") { - if let Some(content_str) = content.as_str() { - let system_content = if let Some(tools) = tools { - // Format tools as a string - let tools_str = serde_json::to_string_pretty(&tools) - .unwrap_or_else(|_| "[]".to_string()); - format!( - "{}\n\nYou have access to the following tools:\n\n{}\n\n\nTo use a tool, respond with a message containing an XML-like function call block like this:\n\n\nvalue\n\n", - content_str, - tools_str - ) - } else { - content_str.to_string() - }; - - *system_message = json!({ - "role": "system", - "content": [{ - "type": "text", - "text": system_content, - "cache_control": { "type": "ephemeral" } - }] - }); - } - } - } - } + {} // Remove any tools/function calling capabilities from the request if let Some(obj) = payload.as_object_mut() {