feat(processing_engine): Add cron plugins and triggers to the process…

…ing engine. (#25852)

* feat(processing_engine): Add cron plugins and triggers to the processing engine.

* feat(processing_engine): switch from 'cron plugin' to 'schedule plugin', use TimeProvider.

* feat(processing_engine): add test for test scheduled plugin.

Cargo.toml

@@ -58,6 +58,7 @@ bitcode = { version = "0.6.3", features = ["serde"] }
 byteorder = "1.3.4"
 bytes = "1.9"
 chrono = "0.4"
+cron = "0.15"
 clap = { version = "4", features = ["derive", "env", "string"] }
 clru = "0.6.2"
 crc32fast = "1.2.0"

influxdb3/src/commands/test.rs

@@ -1,7 +1,7 @@
 use crate::commands::common::{InfluxDb3Config, SeparatedKeyValue, SeparatedList};
 use anyhow::Context;
 use hashbrown::HashMap;
-use influxdb3_client::plugin_development::WalPluginTestRequest;
+use influxdb3_client::plugin_development::{SchedulePluginTestRequest, WalPluginTestRequest};
 use influxdb3_client::Client;
 use secrecy::ExposeSecret;
 use std::error::Error;
@@ -23,6 +23,15 @@ impl Config {
                         ..
                     },
                 ..
+            })
+            | SubCommand::SchedulePlugin(SchedulePluginConfig {
+                influxdb3_config:
+                    InfluxDb3Config {
+                        host_url,
+                        auth_token,
+                        ..
+                    },
+                ..
             }) => {
                 let mut client = Client::new(host_url.clone())?;
                 if let Some(token) = &auth_token {
@@ -39,6 +48,9 @@ pub enum SubCommand {
     /// Test a WAL Plugin
     #[clap(name = "wal_plugin")]
     WalPlugin(WalPluginConfig),
+    /// Test a Cron Plugin
+    #[clap(name = "schedule_plugin")]
+    SchedulePlugin(SchedulePluginConfig),
 }
 
 #[derive(Debug, clap::Parser)]
@@ -60,6 +72,22 @@ pub struct WalPluginConfig {
     pub filename: String,
 }
 
+#[derive(Debug, clap::Parser)]
+pub struct SchedulePluginConfig {
+    #[clap(flatten)]
+    influxdb3_config: InfluxDb3Config,
+    /// If given pass this map of string key/value pairs as input arguments
+    #[clap(long = "input-arguments")]
+    pub input_arguments: Option<SeparatedList<SeparatedKeyValue<String, String>>>,
+    /// The file name of the plugin, which should exist on the server in `<plugin-dir>/<filename>`.
+    /// The plugin-dir is provided on server startup.
+    #[clap(required = true)]
+    pub filename: String,
+    /// Cron schedule to test against. If not given will use * * * * *
+    #[clap(long = "schedule")]
+    pub schedule: Option<String>,
+}
+
 pub async fn command(config: Config) -> Result<(), Box<dyn Error>> {
     let client = config.get_client()?;
 
@@ -96,6 +124,28 @@ pub async fn command(config: Config) -> Result<(), Box<dyn Error>> {
                     .expect("serialize wal plugin test response as JSON")
             );
         }
+        SubCommand::SchedulePlugin(plugin_config) => {
+            let input_arguments = plugin_config.input_arguments.map(|a| {
+                a.into_iter()
+                    .map(|SeparatedKeyValue((k, v))| (k, v))
+                    .collect::<HashMap<String, String>>()
+            });
+            let cron_plugin_test_request = SchedulePluginTestRequest {
+                filename: plugin_config.filename,
+                database: plugin_config.influxdb3_config.database_name,
+                schedule: plugin_config.schedule,
+                input_arguments,
+            };
+            let response = client
+                .schedule_plugin_test(cron_plugin_test_request)
+                .await?;
+
+            println!(
+                "{}",
+                serde_json::to_string_pretty(&response)
+                    .expect("serialize cron plugin test response as JSON")
+            );
+        }
     }
 
     Ok(())

influxdb3/tests/server/cli.rs

@@ -15,7 +15,8 @@ use test_helpers::tempfile::NamedTempFile;
 use test_helpers::tempfile::TempDir;
 use test_helpers::{assert_contains, assert_not_contains};
 
-const WRITE_REPORTS_PLUGIN_CODE: &str = r#"
+#[cfg(feature = "system-py")]
+pub const WRITE_REPORTS_PLUGIN_CODE: &str = r#"
 def process_writes(influxdb3_local, table_batches, args=None):
     for table_batch in table_batches:
         # Skip if table_name is write_reports
@@ -1279,6 +1280,79 @@ def process_writes(influxdb3_local, table_batches, args=None):
     let expected_result = serde_json::from_str::<serde_json::Value>(expected_result).unwrap();
     assert_eq!(res, expected_result);
 }
+#[cfg(feature = "system-py")]
+#[test_log::test(tokio::test)]
+async fn test_schedule_plugin_test() {
+    use crate::ConfigProvider;
+    use influxdb3_client::Precision;
+
+    // Create plugin file with a scheduled task
+    let plugin_file = create_plugin_file(
+        r#"
+def process_scheduled_call(influxdb3_local, schedule_time, args=None):
+    influxdb3_local.info(f"args are {args}")
+    influxdb3_local.info("Successfully called")"#,
+    );
+
+    let plugin_dir = plugin_file.path().parent().unwrap().to_str().unwrap();
+    let plugin_name = plugin_file.path().file_name().unwrap().to_str().unwrap();
+
+    let server = TestServer::configure()
+        .with_plugin_dir(plugin_dir)
+        .spawn()
+        .await;
+    let server_addr = server.client_addr();
+
+    // Write some test data
+    server
+        .write_lp_to_db(
+            "foo",
+            "cpu,host=host1,region=us-east usage=0.75\n\
+             cpu,host=host2,region=us-west usage=0.82\n\
+             cpu,host=host3,region=us-east usage=0.91",
+            Precision::Nanosecond,
+        )
+        .await
+        .unwrap();
+
+    let db_name = "foo";
+
+    // Run the schedule plugin test
+    let result = run_with_confirmation(&[
+        "test",
+        "schedule_plugin",
+        "--database",
+        db_name,
+        "--host",
+        &server_addr,
+        "--schedule",
+        "*/5 * * * * *", // Run every 5 seconds
+        "--input-arguments",
+        "region=us-east",
+        plugin_name,
+    ]);
+    debug!(result = ?result, "test schedule plugin");
+
+    let res = serde_json::from_str::<Value>(&result).unwrap();
+
+    // The trigger_time will be dynamic, so we'll just verify it exists and is in the right format
+    let trigger_time = res["trigger_time"].as_str().unwrap();
+    assert!(trigger_time.contains('T')); // Basic RFC3339 format check
+
+    // Check the rest of the response structure
+    let expected_result = serde_json::json!({
+        "log_lines": [
+            "INFO: args are {'region': 'us-east'}",
+            "INFO: Successfully called"
+        ],
+        "database_writes": {
+        },
+        "errors": []
+    });
+    assert_eq!(res["log_lines"], expected_result["log_lines"]);
+    assert_eq!(res["database_writes"], expected_result["database_writes"]);
+    assert_eq!(res["errors"], expected_result["errors"]);
+}
 
 #[cfg(feature = "system-py")]
 #[test_log::test(tokio::test)]

influxdb3_client/src/lib.rs

@@ -1,6 +1,9 @@
 pub mod plugin_development;
 
-use crate::plugin_development::{WalPluginTestRequest, WalPluginTestResponse};
+use crate::plugin_development::{
+    SchedulePluginTestRequest, SchedulePluginTestResponse, WalPluginTestRequest,
+    WalPluginTestResponse,
+};
 use bytes::Bytes;
 use hashbrown::HashMap;
 use iox_query_params::StatementParam;
@@ -736,6 +739,36 @@ impl Client {
         }
     }
 
+    /// Make a request to the `POST /api/v3/plugin_test/schedule` API
+    pub async fn schedule_plugin_test(
+        &self,
+        schedule_plugin_test_request: SchedulePluginTestRequest,
+    ) -> Result<SchedulePluginTestResponse> {
+        let api_path = "/api/v3/plugin_test/schedule";
+        let url = self.base_url.join(api_path)?;
+
+        let mut req = self
+            .http_client
+            .post(url)
+            .json(&schedule_plugin_test_request);
+        if let Some(token) = &self.auth_token {
+            req = req.bearer_auth(token.expose_secret());
+        }
+        let resp = req
+            .send()
+            .await
+            .map_err(|src| Error::request_send(Method::POST, api_path, src))?;
+
+        if resp.status().is_success() {
+            resp.json().await.map_err(Error::Json)
+        } else {
+            Err(Error::ApiError {
+                code: resp.status(),
+                message: resp.text().await.map_err(Error::Text)?,
+            })
+        }
+    }
+
     /// Send a `/ping` request to the target `influxdb3` server to check its
     /// status and gather `version` and `revision` information
     pub async fn ping(&self) -> Result<PingResponse> {

influxdb3_client/src/plugin_development.rs

@@ -19,3 +19,21 @@ pub struct WalPluginTestResponse {
     pub database_writes: HashMap<String, Vec<String>>,
     pub errors: Vec<String>,
 }
+
+/// Request definition for `POST /api/v3/plugin_test/schedule` API
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SchedulePluginTestRequest {
+    pub filename: String,
+    pub database: String,
+    pub schedule: Option<String>,
+    pub input_arguments: Option<HashMap<String, String>>,
+}
+
+/// Response definition for `POST /api/v3/plugin_test/schedule` API
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SchedulePluginTestResponse {
+    pub trigger_time: Option<String>,
+    pub log_lines: Vec<String>,
+    pub database_writes: HashMap<String, Vec<String>>,
+    pub errors: Vec<String>,
+}

influxdb3_processing_engine/Cargo.toml

@@ -11,6 +11,8 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 async-trait.workspace = true
+chrono.workspace = true
+cron.workspace =  true
 data_types.workspace = true
 hashbrown.workspace = true
 iox_time.workspace = true