Update librdkafka to 2.6.0

rdkafka-sys/CONTRIBUTING.md

@@ -21,7 +21,7 @@ Then:
 
 * Checkout into master and pull the latest changes.
 * Ensure the changelog is up to date (i.e not Unreleased changes).
-* Run `cd rdkafka-sys && ../generate_readme.py > README.md`.
+* Run `cd rdkafka-sys && ../generate_readme.py > README.md`. 
 * Bump the version in Cargo.toml and commit locally.
 * Run `cargo publish`.
 * Push the commit.

rdkafka-sys/changelog.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+## v4.8.0+2.6.0 (2024-11-17)
+
+* Upgrade to librdkafka v2.3.0.
+* Add cargo enforcement of MSRV of 1.61.
+*
 ## v4.7.0+2.2.0 (2023-11-07)
 
 * Upgrade to librdkafka v2.3.0.

rdkafka-sys/src/bindings.rs

@@ -1,9 +1,9 @@
-/* automatically generated by rust-bindgen 0.66.1 */
+/* automatically generated by rust-bindgen 0.70.1 */
 
 use libc::{c_char, c_int, c_void, sockaddr, FILE};
 use num_enum::TryFromPrimitive;
 
-pub const RD_KAFKA_VERSION: i32 = 33751295;
+pub const RD_KAFKA_VERSION: i32 = 33947903;
 pub const RD_KAFKA_DEBUG_CONTEXTS : & [u8 ; 138] = b"all,generic,broker,topic,metadata,feature,queue,msg,protocol,cgrp,security,fetch,interceptor,plugin,consumer,admin,eos,mock,assignor,conf\0" ;
 pub const RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE: i32 = 8;
 pub const RD_KAFKA_OFFSET_BEGINNING: i32 = -2;
@@ -49,6 +49,7 @@ pub const RD_KAFKA_EVENT_ALTERUSERSCRAMCREDENTIALS_RESULT: i32 = 524288;
 pub const RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT: i32 = 1048576;
 pub const RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT: i32 = 2097152;
 pub const RD_KAFKA_EVENT_LISTOFFSETS_RESULT: i32 = 4194304;
+pub const RD_KAFKA_EVENT_ELECTLEADERS_RESULT: i32 = 8388608;
 extern "C" {
     pub fn rd_kafka_version() -> c_int;
 }
@@ -149,6 +150,12 @@ pub struct rd_kafka_Uuid_s {
     _unused: [u8; 0],
 }
 pub type rd_kafka_Uuid_t = rd_kafka_Uuid_s;
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+pub struct rd_kafka_topic_partition_result_s {
+    _unused: [u8; 0],
+}
+pub type rd_kafka_topic_partition_result_t = rd_kafka_topic_partition_result_s;
 #[repr(i32)]
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, TryFromPrimitive)]
 pub enum rd_kafka_resp_err_t {
@@ -214,6 +221,7 @@ pub enum rd_kafka_resp_err_t {
     RD_KAFKA_RESP_ERR__NOOP = -141,
     RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET = -140,
     RD_KAFKA_RESP_ERR__LOG_TRUNCATION = -139,
+    RD_KAFKA_RESP_ERR__INVALID_DIFFERENT_RECORD = -138,
     RD_KAFKA_RESP_ERR__END = -100,
     RD_KAFKA_RESP_ERR_UNKNOWN = -1,
     RD_KAFKA_RESP_ERR_NO_ERROR = 0,
@@ -314,7 +322,14 @@ pub enum rd_kafka_resp_err_t {
     RD_KAFKA_RESP_ERR_INVALID_UPDATE_VERSION = 95,
     RD_KAFKA_RESP_ERR_FEATURE_UPDATE_FAILED = 96,
     RD_KAFKA_RESP_ERR_PRINCIPAL_DESERIALIZATION_FAILURE = 97,
-    RD_KAFKA_RESP_ERR_END_ALL = 98,
+    RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID = 100,
+    RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH = 110,
+    RD_KAFKA_RESP_ERR_UNRELEASED_INSTANCE_ID = 111,
+    RD_KAFKA_RESP_ERR_UNSUPPORTED_ASSIGNOR = 112,
+    RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH = 113,
+    RD_KAFKA_RESP_ERR_UNKNOWN_SUBSCRIPTION_ID = 117,
+    RD_KAFKA_RESP_ERR_TELEMETRY_TOO_LARGE = 118,
+    RD_KAFKA_RESP_ERR_END_ALL = 119,
 }
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
@@ -604,6 +619,9 @@ extern "C" {
 extern "C" {
     pub fn rd_kafka_message_errstr(rkmessage: *const rd_kafka_message_t) -> *const c_char;
 }
+extern "C" {
+    pub fn rd_kafka_message_produce_errstr(rkmessage: *const rd_kafka_message_t) -> *const c_char;
+}
 extern "C" {
     pub fn rd_kafka_message_timestamp(
         rkmessage: *const rd_kafka_message_t,
@@ -1500,6 +1518,11 @@ extern "C" {
         group_instance_id: *const c_char,
     ) -> *mut rd_kafka_consumer_group_metadata_t;
 }
+extern "C" {
+    pub fn rd_kafka_consumer_group_metadata_member_id(
+        group_metadata: *const rd_kafka_consumer_group_metadata_t,
+    ) -> *const c_char;
+}
 extern "C" {
     pub fn rd_kafka_consumer_group_metadata_destroy(arg1: *mut rd_kafka_consumer_group_metadata_t);
 }
@@ -1646,6 +1669,14 @@ pub enum rd_kafka_consumer_group_state_t {
     RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY = 5,
     RD_KAFKA_CONSUMER_GROUP_STATE__CNT = 6,
 }
+#[repr(u32)]
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub enum rd_kafka_consumer_group_type_t {
+    RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN = 0,
+    RD_KAFKA_CONSUMER_GROUP_TYPE_CONSUMER = 1,
+    RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC = 2,
+    RD_KAFKA_CONSUMER_GROUP_TYPE__CNT = 3,
+}
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct rd_kafka_group_info {
@@ -1682,6 +1713,15 @@ extern "C" {
         name: *const c_char,
     ) -> rd_kafka_consumer_group_state_t;
 }
+extern "C" {
+    pub fn rd_kafka_consumer_group_type_name(
+        type_: rd_kafka_consumer_group_type_t,
+    ) -> *const c_char;
+}
+extern "C" {
+    pub fn rd_kafka_consumer_group_type_code(name: *const c_char)
+        -> rd_kafka_consumer_group_type_t;
+}
 extern "C" {
     pub fn rd_kafka_group_list_destroy(grplist: *const rd_kafka_group_list);
 }
@@ -1832,6 +1872,7 @@ pub type rd_kafka_DescribeCluster_result_t = rd_kafka_event_t;
 pub type rd_kafka_DescribeUserScramCredentials_result_t = rd_kafka_event_t;
 pub type rd_kafka_AlterUserScramCredentials_result_t = rd_kafka_event_t;
 pub type rd_kafka_ListOffsets_result_t = rd_kafka_event_t;
+pub type rd_kafka_ElectLeaders_result_t = rd_kafka_event_t;
 extern "C" {
     pub fn rd_kafka_event_CreateTopics_result(
         rkev: *mut rd_kafka_event_t,
@@ -1937,6 +1978,11 @@ extern "C" {
         rkev: *mut rd_kafka_event_t,
     ) -> *const rd_kafka_AlterUserScramCredentials_result_t;
 }
+extern "C" {
+    pub fn rd_kafka_event_ElectLeaders_result(
+        rkev: *mut rd_kafka_event_t,
+    ) -> *const rd_kafka_ElectLeaders_result_t;
+}
 extern "C" {
     pub fn rd_kafka_queue_poll(
         rkqu: *mut rd_kafka_queue_t,
@@ -2209,6 +2255,16 @@ extern "C" {
         groupres: *const rd_kafka_group_result_t,
     ) -> *const rd_kafka_topic_partition_list_t;
 }
+extern "C" {
+    pub fn rd_kafka_topic_partition_result_partition(
+        partition_result: *const rd_kafka_topic_partition_result_t,
+    ) -> *const rd_kafka_topic_partition_t;
+}
+extern "C" {
+    pub fn rd_kafka_topic_partition_result_error(
+        partition_result: *const rd_kafka_topic_partition_result_t,
+    ) -> *const rd_kafka_error_t;
+}
 #[repr(u32)]
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub enum rd_kafka_admin_op_t {
@@ -2234,7 +2290,8 @@ pub enum rd_kafka_admin_op_t {
     RD_KAFKA_ADMIN_OP_DESCRIBETOPICS = 19,
     RD_KAFKA_ADMIN_OP_DESCRIBECLUSTER = 20,
     RD_KAFKA_ADMIN_OP_LISTOFFSETS = 21,
-    RD_KAFKA_ADMIN_OP__CNT = 22,
+    RD_KAFKA_ADMIN_OP_ELECTLEADERS = 22,
+    RD_KAFKA_ADMIN_OP__CNT = 23,
 }
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
@@ -2308,6 +2365,13 @@ extern "C" {
         consumer_group_states_cnt: usize,
     ) -> *mut rd_kafka_error_t;
 }
+extern "C" {
+    pub fn rd_kafka_AdminOptions_set_match_consumer_group_types(
+        options: *mut rd_kafka_AdminOptions_t,
+        consumer_group_types: *const rd_kafka_consumer_group_type_t,
+        consumer_group_types_cnt: usize,
+    ) -> *mut rd_kafka_error_t;
+}
 extern "C" {
     pub fn rd_kafka_AdminOptions_set_isolation_level(
         options: *mut rd_kafka_AdminOptions_t,
@@ -2532,7 +2596,8 @@ pub enum rd_kafka_ResourceType_t {
     RD_KAFKA_RESOURCE_TOPIC = 2,
     RD_KAFKA_RESOURCE_GROUP = 3,
     RD_KAFKA_RESOURCE_BROKER = 4,
-    RD_KAFKA_RESOURCE__CNT = 5,
+    RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5,
+    RD_KAFKA_RESOURCE__CNT = 6,
 }
 #[repr(u32)]
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
@@ -2858,6 +2923,11 @@ extern "C" {
         grplist: *const rd_kafka_ConsumerGroupListing_t,
     ) -> rd_kafka_consumer_group_state_t;
 }
+extern "C" {
+    pub fn rd_kafka_ConsumerGroupListing_type(
+        grplist: *const rd_kafka_ConsumerGroupListing_t,
+    ) -> rd_kafka_consumer_group_type_t;
+}
 extern "C" {
     pub fn rd_kafka_ListConsumerGroups_result_valid(
         result: *const rd_kafka_ListConsumerGroups_result_t,
@@ -3453,6 +3523,41 @@ extern "C" {
         rkqu: *mut rd_kafka_queue_t,
     );
 }
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+pub struct rd_kafka_ElectLeaders_s {
+    _unused: [u8; 0],
+}
+pub type rd_kafka_ElectLeaders_t = rd_kafka_ElectLeaders_s;
+#[repr(u32)]
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub enum rd_kafka_ElectionType_t {
+    RD_KAFKA_ELECTION_TYPE_PREFERRED = 0,
+    RD_KAFKA_ELECTION_TYPE_UNCLEAN = 1,
+}
+extern "C" {
+    pub fn rd_kafka_ElectLeaders_new(
+        election_type: rd_kafka_ElectionType_t,
+        partitions: *mut rd_kafka_topic_partition_list_t,
+    ) -> *mut rd_kafka_ElectLeaders_t;
+}
+extern "C" {
+    pub fn rd_kafka_ElectLeaders_destroy(elect_leaders: *mut rd_kafka_ElectLeaders_t);
+}
+extern "C" {
+    pub fn rd_kafka_ElectLeaders(
+        rk: *mut rd_kafka_t,
+        elect_leaders: *mut rd_kafka_ElectLeaders_t,
+        options: *const rd_kafka_AdminOptions_t,
+        rkqu: *mut rd_kafka_queue_t,
+    );
+}
+extern "C" {
+    pub fn rd_kafka_ElectLeaders_result_partitions(
+        result: *const rd_kafka_ElectLeaders_result_t,
+        cntp: *mut usize,
+    ) -> *mut *const rd_kafka_topic_partition_result_t;
+}
 extern "C" {
     pub fn rd_kafka_oauthbearer_set_token(
         rk: *mut rd_kafka_t,
@@ -3547,6 +3652,12 @@ extern "C" {
         errors: *const rd_kafka_resp_err_t,
     );
 }
+extern "C" {
+    pub fn rd_kafka_mock_group_initial_rebalance_delay_ms(
+        mcluster: *mut rd_kafka_mock_cluster_t,
+        delay_ms: i32,
+    );
+}
 extern "C" {
     pub fn rd_kafka_mock_broker_push_request_error_rtts(
         mcluster: *mut rd_kafka_mock_cluster_t,
@@ -3604,6 +3715,15 @@ extern "C" {
         hi: i64,
     ) -> rd_kafka_resp_err_t;
 }
+extern "C" {
+    pub fn rd_kafka_mock_partition_push_leader_response(
+        mcluster: *mut rd_kafka_mock_cluster_t,
+        topic: *const c_char,
+        partition: c_int,
+        leader_id: i32,
+        leader_epoch: i32,
+    ) -> rd_kafka_resp_err_t;
+}
 extern "C" {
     pub fn rd_kafka_mock_broker_set_down(
         mcluster: *mut rd_kafka_mock_cluster_t,
@@ -3661,6 +3781,12 @@ pub type rd_kafka_mock_request_t = rd_kafka_mock_request_s;
 extern "C" {
     pub fn rd_kafka_mock_request_destroy(mreq: *mut rd_kafka_mock_request_t);
 }
+extern "C" {
+    pub fn rd_kafka_mock_request_destroy_array(
+        mreqs: *mut *mut rd_kafka_mock_request_t,
+        mreq_cnt: usize,
+    );
+}
 extern "C" {
     pub fn rd_kafka_mock_request_id(mreq: *mut rd_kafka_mock_request_t) -> i32;
 }
@@ -3679,3 +3805,16 @@ extern "C" {
 extern "C" {
     pub fn rd_kafka_mock_clear_requests(mcluster: *mut rd_kafka_mock_cluster_t);
 }
+extern "C" {
+    pub fn rd_kafka_mock_telemetry_set_requested_metrics(
+        mcluster: *mut rd_kafka_mock_cluster_t,
+        metrics: *mut *mut c_char,
+        metrics_cnt: usize,
+    ) -> rd_kafka_resp_err_t;
+}
+extern "C" {
+    pub fn rd_kafka_mock_telemetry_set_push_interval(
+        mcluster: *mut rd_kafka_mock_cluster_t,
+        push_interval_ms: i64,
+    ) -> rd_kafka_resp_err_t;
+}

rdkafka-sys/src/helpers.rs

@@ -68,6 +68,8 @@ pub fn rd_kafka_resp_err_t_to_rdkafka_error(err: RDKafkaRespErr) -> RDKafkaError
         RD_KAFKA_RESP_ERR__ASSIGNMENT_LOST => AssignmentLost,
         RD_KAFKA_RESP_ERR__NOOP => Noop,
         RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET => AutoOffsetReset,
+        RD_KAFKA_RESP_ERR__LOG_TRUNCATION => LogTruncation,
+        RD_KAFKA_RESP_ERR__INVALID_DIFFERENT_RECORD => InvalidDifferentRecord,
         RD_KAFKA_RESP_ERR__END => End,
         RD_KAFKA_RESP_ERR_UNKNOWN => Unknown,
         RD_KAFKA_RESP_ERR_NO_ERROR => NoError,
@@ -172,7 +174,13 @@ pub fn rd_kafka_resp_err_t_to_rdkafka_error(err: RDKafkaRespErr) -> RDKafkaError
         RD_KAFKA_RESP_ERR_INVALID_UPDATE_VERSION => InvalidUpdateVersion,
         RD_KAFKA_RESP_ERR_FEATURE_UPDATE_FAILED => FeatureUpdateFailed,
         RD_KAFKA_RESP_ERR_PRINCIPAL_DESERIALIZATION_FAILURE => PrincipalDeserializationFailure,
+        RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID => UnknownTopicId,
+        RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH => FencedMemberEpoch,
+        RD_KAFKA_RESP_ERR_UNRELEASED_INSTANCE_ID => UnreleasedInstanceId,
+        RD_KAFKA_RESP_ERR_UNSUPPORTED_ASSIGNOR => UnsupportedAssignor,
+        RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH => StaleMemberEpoch,
+        RD_KAFKA_RESP_ERR_UNKNOWN_SUBSCRIPTION_ID => UnknownSubscriptionId,
+        RD_KAFKA_RESP_ERR_TELEMETRY_TOO_LARGE => TelemetryTooLarge,
         RD_KAFKA_RESP_ERR_END_ALL => EndAll,
-        RD_KAFKA_RESP_ERR__LOG_TRUNCATION => LogTruncation,
     }
 }

rdkafka-sys/src/types.rs

@@ -249,6 +249,8 @@ pub enum RDKafkaErrorCode {
     AutoOffsetReset = -140,
     /// Partition log truncation detected
     LogTruncation = -139,
+    /// A different record in the batch was invalid and this message failed persisting.
+    InvalidDifferentRecord = -138,
     #[doc(hidden)]
     End = -100,
     /// Unknown broker error.
@@ -456,6 +458,20 @@ pub enum RDKafkaErrorCode {
     FeatureUpdateFailed = 96,
     /// Request principal deserialization failed during forwarding.
     PrincipalDeserializationFailure = 97,
+    /// Unknown Topic Id
+    UnknownTopicId = 100,
+    /// The member epoch is fenced by the group coordinator
+    FencedMemberEpoch = 110,
+    /// The instance ID is still used by another member in the consumer group
+    UnreleasedInstanceId = 111,
+    /// The assignor or its version range is not supported by the consumer group
+    UnsupportedAssignor = 112,
+    /// The member epoch is stale
+    StaleMemberEpoch = 113,
+    /// Client sent a push telemetry request with an invalid or outdated subscription ID.
+    UnknownSubscriptionId = 117,
+    /// Client sent a push telemetry request larger than the maximum size the broker will accept.
+    TelemetryTooLarge = 118,
     #[doc(hidden)]
     EndAll,
 }